From fbbcd047ce3add89d8227b23f15c58c5993366b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABl=20de=20Chalendar?= <gael.de-chalendar@cea.fr>
Date: Wed, 8 May 2024 00:19:41 +0200
Subject: [PATCH] Add pipeline forging

---
 lima_common/conf/lima-analysis.xml            | 80 ++++++++++++++++--
 .../ProcessUnitPipeline.h                     |  1 +
 .../ProcessUnitPipeline.tcc                   |  6 ++
 .../conf/lima-lp-ud-eng.xml                   |  2 +-
 lima_linguisticprocessing/conf/lima-lp-ud.xml |  2 +-
 .../client/CMakeLists.txt                     | 74 ++++++++++++++++
 .../linguisticProcessing/client}/lima.cpp     | 75 ++++++++++++++++-
 .../linguisticProcessing/client}/lima.h       |  4 +
 .../core/CoreLinguisticProcessingClient.h     | 10 ++-
 .../test/analyzeText.cpp                      | 84 ++++++++++++++++++-
 .../tests/CMakeLists.txt                      | 71 +---------------
 .../tests/LimaAnalyzerTest0.cpp               | 51 ++++++++++-
 .../tests/LimaAnalyzerTest0.h                 |  5 +-
 lima_linguisticprocessing/tests/test_lima.cpp | 67 ---------------
 14 files changed, 377 insertions(+), 155 deletions(-)
 rename lima_linguisticprocessing/{tests => src/linguisticProcessing/client}/lima.cpp (93%)
 rename lima_linguisticprocessing/{tests => src/linguisticProcessing/client}/lima.h (95%)
 delete mode 100644 lima_linguisticprocessing/tests/test_lima.cpp

diff --git a/lima_common/conf/lima-analysis.xml b/lima_common/conf/lima-analysis.xml
index d762c45bc..1a05953ec 100644
--- a/lima_common/conf/lima-analysis.xml
+++ b/lima_common/conf/lima-analysis.xml
@@ -511,11 +511,81 @@ SPDX-License-Identifier: MIT
         <entry key="fre" value="xml"/>
         <entry key="por" value="xml"/>
       </map>
-      <!-- The none pipeline is an empty one allowing to not do anything. For tests mainly. -->
-      <map name="none">
-        <entry key="eng" value="none"/>
-        <entry key="fre" value="none"/>
-        <entry key="por" value="none"/>
+      <!-- The empty pipeline is an empty one allowing to not do anything. For tests mainly. -->
+      <map name="empty">
+        <entry key="eng"    value="empty"/>
+        <entry key="fre"    value="empty"/>
+        <entry key="por"    value="empty"/>
+
+        <entry key="ud"     value="empty"/>
+
+        <entry key="ud-afr" value="empty" />
+        <entry key="ud-ara" value="empty" />
+        <entry key="ud-bel" value="empty" />
+        <entry key="ud-bre" value="empty" />
+        <entry key="ud-bul" value="empty" />
+        <entry key="ud-cat" value="empty" />
+        <entry key="ud-ces" value="empty" />
+        <entry key="ud-chu" value="empty" />
+        <entry key="ud-cop" value="empty" />
+        <entry key="ud-cym" value="empty" />
+        <entry key="ud-dan" value="empty" />
+        <entry key="ud-deu" value="empty" />
+        <entry key="ud-ell" value="empty" />
+        <entry key="ud-eng" value="empty" />
+        <entry key="ud-est" value="empty" />
+        <entry key="ud-eus" value="empty" />
+        <entry key="ud-fas" value="empty" />
+        <entry key="ud-fin" value="empty" />
+        <entry key="ud-fra" value="empty" />
+        <entry key="ud-fro" value="empty" />
+        <entry key="ud-gla" value="empty" />
+        <entry key="ud-gle" value="empty" />
+        <entry key="ud-glg" value="empty" />
+        <entry key="ud-heb" value="empty" />
+        <entry key="ud-hin" value="empty" />
+        <entry key="ud-hrv" value="empty" />
+        <entry key="ud-hun" value="empty" />
+        <entry key="ud-hye" value="empty" />
+        <entry key="ud-ind" value="empty" />
+        <entry key="ud-ita" value="empty" />
+        <entry key="ud-jpn" value="empty" />
+        <entry key="ud-kaz" value="empty" />
+        <entry key="ud-kmr" value="empty" />
+        <entry key="ud-kor" value="empty" />
+        <entry key="ud-lat" value="empty" />
+        <entry key="ud-lav" value="empty" />
+        <entry key="ud-lit" value="empty" />
+        <entry key="ud-lzh" value="empty" />
+        <entry key="ud-mar" value="empty" />
+        <entry key="ud-mlt" value="empty" />
+        <entry key="ud-nld" value="empty" />
+        <entry key="ud-nno" value="empty" />
+        <entry key="ud-nob" value="empty" />
+        <entry key="ud-orv" value="empty" />
+        <entry key="ud-pcm" value="empty" />
+        <entry key="ud-pol" value="empty" />
+        <entry key="ud-por" value="empty" />
+        <entry key="ud-ron" value="empty" />
+        <entry key="ud-rus" value="empty" />
+        <entry key="ud-slk" value="empty" />
+        <entry key="ud-slv" value="empty" />
+        <entry key="ud-sme" value="empty" />
+        <entry key="ud-spa" value="empty" />
+        <entry key="ud-sqi" value="empty" />
+        <entry key="ud-srp" value="empty" />
+        <entry key="ud-swe" value="empty" />
+        <entry key="ud-tam" value="empty" />
+        <entry key="ud-tel" value="empty" />
+        <entry key="ud-tha" value="empty" />
+        <entry key="ud-tur" value="empty" />
+        <entry key="ud-uig" value="empty" />
+        <entry key="ud-ukr" value="empty" />
+        <entry key="ud-urd" value="empty" />
+        <entry key="ud-vie" value="empty" />
+        <entry key="ud-wol" value="empty" />
+        <entry key="ud-zho-simp" value="empty" />
+        <entry key="ud-zho" value="empty" />
       </map>
     </group>
     <group name="analysisDumpers">
diff --git a/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.h b/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.h
index 13345f573..8cea4436e 100644
--- a/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.h
+++ b/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.h
@@ -63,6 +63,7 @@ class ProcessUnitPipeline : public ProcessUnit
     const std::string& unitId,
     typename ProcessUnit::Manager* manager);
 
+  void push_back(std::shared_ptr<ProcessUnit> pu);
 
 private:
   void debugPrintInactiveUnits() const;
diff --git a/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.tcc b/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.tcc
index dca4fbfc8..7731641b1 100644
--- a/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.tcc
+++ b/lima_common/src/common/ProcessUnitFramework/ProcessUnitPipeline.tcc
@@ -55,6 +55,12 @@ void ProcessUnitPipeline<ProcessUnit>::push_back(
   m_processUnitSequence.push_back(pu);
 }
 
+template <typename ProcessUnit>
+void ProcessUnitPipeline<ProcessUnit>::push_back(std::shared_ptr<ProcessUnit> pu)
+{
+  m_processUnitSequence.push_back(pu);
+}
+
 template <typename ProcessUnit>
 LimaStatusCode ProcessUnitPipeline<ProcessUnit>::process(
     AnalysisContent& analysis) const
diff --git a/lima_linguisticprocessing/conf/lima-lp-ud-eng.xml b/lima_linguisticprocessing/conf/lima-lp-ud-eng.xml
index 3ca21a826..5d33cc52d 100644
--- a/lima_linguisticprocessing/conf/lima-lp-ud-eng.xml
+++ b/lima_linguisticprocessing/conf/lima-lp-ud-eng.xml
@@ -80,7 +80,7 @@ SPDX-License-Identifier: MIT
         </list>
     </group>
 
-    <group name="none" class="ProcessUnitPipeline">
+    <group name="empty" class="ProcessUnitPipeline">
       <list name="processUnitSequence"/>
     </group>
 <!-- ******************************************
diff --git a/lima_linguisticprocessing/conf/lima-lp-ud.xml b/lima_linguisticprocessing/conf/lima-lp-ud.xml
index 9ca989960..173b0665a 100644
--- a/lima_linguisticprocessing/conf/lima-lp-ud.xml
+++ b/lima_linguisticprocessing/conf/lima-lp-ud.xml
@@ -68,7 +68,7 @@ SPDX-License-Identifier: MIT
       </list>
     </group>
 
-    <group name="none" class="ProcessUnitPipeline">
+    <group name="empty" class="ProcessUnitPipeline">
       <list name="processUnitSequence"/>
     </group>
 <!-- ******************************************
diff --git a/lima_linguisticprocessing/src/linguisticProcessing/client/CMakeLists.txt b/lima_linguisticprocessing/src/linguisticProcessing/client/CMakeLists.txt
index 1933191e1..8eb4ce7fe 100644
--- a/lima_linguisticprocessing/src/linguisticProcessing/client/CMakeLists.txt
+++ b/lima_linguisticprocessing/src/linguisticProcessing/client/CMakeLists.txt
@@ -10,6 +10,80 @@ add_subdirectory(xmlreader)
 
 add_definitions(-DLIMA_LINGUISTICPROCESSIONGCLIENT_EXPORTING)
 
+########### next target ###############
+
+SET(lima-lp-analyzer_LIB_SRCS
+    lima.cpp
+)
+
+add_library(lima-lp-analyzer SHARED ${lima-lp-analyzer_LIB_SRCS})
+ADD_DLL_TO_PATH(lima-lp-analyzer)
+
+
+target_link_libraries(lima-lp-analyzer
+    lima-linguisticprocessing
+    lima-common-datahandler
+    lima-common-data
+    lima-common-factory
+    lima-common-fsaaccess
+    lima-common-mediaprocessors
+    lima-common-mediaticdata
+    lima-common-misc
+    lima-common-processunitframework
+    lima-common
+    lima-common-time
+    lima-common-tools
+    lima-common-xmlconfigurationfiles
+    lima-linguisticprocessing
+    lima-lp-analysisdict
+    lima-lp-analysisdumpers
+    lima-lp-analysishandlers
+    lima-lp-applyrecognizer
+    lima-lp-automatoncompiler
+    lima-lp-automaton
+    lima-lp-bagofwords
+    lima-lp-client
+    lima-lp-compounds
+    lima-lp-conllureader
+    lima-lp-corefsolver
+    lima-lp-dictionary
+    lima-lp-entitytracker
+    lima-lp-eventanalyzer
+    lima-lp-flattokenizer
+    lima-lp-helpers
+    lima-lp-langdetector
+    lima-lp-lineartextrepresentation
+    lima-lp-linguisticanalysisstructure
+    lima-lp-linguisticdata
+    lima-lp-linguisticprocessing-core
+    lima-lp-linguisticprocessors
+    lima-lp-linguisticresources
+    lima-lp-misc
+    lima-lp-morphologicanalysis
+    lima-lp-postagger
+    lima-lp-propertycode
+    lima-lp-regexmatcher
+    lima-lp-se-datetime
+    lima-lp-semanticanalysis
+    lima-lp-se-number
+    lima-lp-se-person
+    lima-lp-specificentities
+    lima-lp-syntacticanalysis
+    lima-lp-textsegmentation
+    lima-lp-tgv
+    lima-lp-wordsensetagger
+    lima-xp-client
+    lima-xp-core
+    lima-xp-documentsreader
+    lima-xp-qtsgmlentities
+)
+
+set_target_properties(lima-lp-analyzer PROPERTIES VERSION ${LIMA_LP_LIB_VERSION} SOVERSION ${LIMA_LP_LIB_SOVERSION})
+
+install(TARGETS lima-lp-analyzer DESTINATION ${LIB_INSTALL_DIR}
+  COMPONENT runtime)
+
+
 ########### next target ###############
 
 SET(lima-lp-client_LIB_SRCS
diff --git a/lima_linguisticprocessing/tests/lima.cpp b/lima_linguisticprocessing/src/linguisticProcessing/client/lima.cpp
similarity index 93%
rename from lima_linguisticprocessing/tests/lima.cpp
rename to lima_linguisticprocessing/src/linguisticProcessing/client/lima.cpp
index df8c13bab..c391d8edf 100644
--- a/lima_linguisticprocessing/tests/lima.cpp
+++ b/lima_linguisticprocessing/src/linguisticProcessing/client/lima.cpp
@@ -59,6 +59,7 @@
 #include "common/LimaVersion.h"
 #include "common/Data/strwstrtools.h"
 #include "common/MediaticData/mediaticData.h"
+#include "common/MediaProcessors/MediaProcessors.h"
 #include "common/MediaProcessors/MediaProcessUnit.h"
 #include <common/ProcessUnitFramework/AnalysisContent.h>
 #include "common/QsLog/QsLog.h"
@@ -66,6 +67,7 @@
 #include "common/QsLog/QsLogCategories.h"
 #include "common/QsLog/QsDebugOutput.h"
 #include "common/XMLConfigurationFiles/xmlConfigurationFileParser.h"
+#include "common/XMLConfigurationFiles/groupConfigurationStructure.h"
 #include "common/time/traceUtils.h"
 #include "common/tools/FileUtils.h"
 #include "common/tools/LimaMainTaskRunner.h"
@@ -117,6 +119,7 @@ using LangData = Lima::Common::MediaticData::LanguageData;
 using MedData = Lima::Common::MediaticData::MediaticData ;
 using namespace Lima::Common::Misc;
 using namespace Lima::Common::PropertyCode;
+using namespace Lima::Common::XMLConfigurationFiles;
 using namespace Lima;
 
 struct character_escaper
@@ -157,6 +160,10 @@ class LimaAnalyzerPrivate
 
   void initMetaData();
 
+  bool addPipelineUnit(const std::string& pipeline,
+                                            const std::string& media,
+                                            const std::string& jsonGroupString);
+
   const std::string analyzeText(const std::string& text,
                                 const std::string& lang,
                                 const std::string& pipeline,
@@ -184,6 +191,7 @@ class LimaAnalyzerPrivate
   /** Reset all members used to store analysis states. To be called before handling a new analysis. */
   void reset();
 
+  QJsonObject objectFromString(const QString& in);
 
   QString previousNeType;
 
@@ -324,7 +332,7 @@ LimaAnalyzerPrivate::LimaAnalyzerPrivate(const QStringList& iqlangs,
       //     << clientId << std::endl;
 
       // initialize linguistic processing
-      Lima::Common::XMLConfigurationFiles::XMLConfigurationFileParser lpconfig(
+      XMLConfigurationFileParser lpconfig(
           (configDir + "/" + lpConfigFile.c_str()));
       LinguisticProcessingClientFactory::changeable().configureClientFactory(
         clientId,
@@ -475,6 +483,71 @@ void LimaAnalyzerPrivate::initMetaData ()
   }
 }
 
+bool LimaAnalyzer::addPipelineUnit(const std::string& pipeline,
+                     const std::string& media,
+                     const std::string& jsonGroupString)
+{
+  if (! m_d->addPipelineUnit(pipeline, media, jsonGroupString))
+  {
+    error = true;
+    errorMessage = "addPipelineUnit: failed";
+    return false;
+  }
+  return true;
+}
+
+QJsonObject LimaAnalyzerPrivate::objectFromString(const QString& in)
+{
+    QJsonObject obj;
+
+    QJsonDocument doc = QJsonDocument::fromJson(in.toUtf8());
+
+    // check validity of the document
+    if(!doc.isNull())
+    {
+        if(doc.isObject())
+        {
+            obj = doc.object();
+        }
+        else
+        {
+            qDebug() << "Document is not an object" << Qt::endl;
+        }
+    }
+    else
+    {
+        qDebug() << "Invalid JSON...\n" << in << Qt::endl;
+    }
+
+    return obj;
+}
+
+bool LimaAnalyzerPrivate::addPipelineUnit(const std::string& pipeline,
+                                          const std::string& media,
+                                          const std::string& jsonGroupString)
+{
+  auto jsonGroup = QJsonDocument::fromJson(QByteArray::fromStdString(jsonGroupString)).object();
+  auto mediaid = Lima::Common::MediaticData::MediaticData::single().getMediaId(media);
+  auto pipe = MediaProcessors::changeable().getPipelineForId(mediaid, pipeline);
+  auto managers = Lima::MediaProcessors::single().managers();
+    // QString jsonGroupString =
+    //     "{ \"name\":\"cpptftokenizer\", "
+    //     "  \"class\":\"CppUppsalaTensorFlowTokenizer\", "
+    //     "  \"model_prefix\": \"tokenizer-eng\" }";
+
+  GroupConfigurationStructure unitConfig(jsonGroup);
+
+  // managers (Manager*) used in init of ProcessUnits are stored in
+  // m_pipelineManagers of MediaProcessors
+  auto pu = MediaProcessUnit::Factory::getFactory(
+    jsonGroup["class"].toString().toStdString())->create(
+      unitConfig,
+      managers[mediaid]);
+  pipe->push_back(pu);
+
+  return true;
+}
+
 std::string LimaAnalyzer::analyzeText(const std::string& text,
                                     const std::string& lang,
                                     const std::string& pipeline,
diff --git a/lima_linguisticprocessing/tests/lima.h b/lima_linguisticprocessing/src/linguisticProcessing/client/lima.h
similarity index 95%
rename from lima_linguisticprocessing/tests/lima.h
rename to lima_linguisticprocessing/src/linguisticProcessing/client/lima.h
index fd9456da7..82c1d70e8 100644
--- a/lima_linguisticprocessing/tests/lima.h
+++ b/lima_linguisticprocessing/src/linguisticProcessing/client/lima.h
@@ -82,6 +82,10 @@ class LimaAnalyzer
                           const std::string& pipeline="",
                           const std::string& meta="");
 
+  bool addPipelineUnit(const std::string& pipeline,
+                       const std::string& media,
+                       const std::string& jsonGroupString);
+
   bool error = false;
   std::string errorMessage = "";
 
diff --git a/lima_linguisticprocessing/src/linguisticProcessing/core/CoreLinguisticProcessingClient.h b/lima_linguisticprocessing/src/linguisticProcessing/core/CoreLinguisticProcessingClient.h
index 010d2fbaa..0204757d6 100644
--- a/lima_linguisticprocessing/src/linguisticProcessing/core/CoreLinguisticProcessingClient.h
+++ b/lima_linguisticprocessing/src/linguisticProcessing/core/CoreLinguisticProcessingClient.h
@@ -22,7 +22,8 @@ namespace LinguisticProcessing
 /**
 @author Benoit Mathieu
 */
-class LIMA_CORELINGUISTICPROCESSINGCLIENT_EXPORT CoreLinguisticProcessingClient : public AbstractLinguisticProcessingClient
+class LIMA_CORELINGUISTICPROCESSINGCLIENT_EXPORT CoreLinguisticProcessingClient :
+    public AbstractLinguisticProcessingClient
 {
 public:
   CoreLinguisticProcessingClient();
@@ -34,15 +35,16 @@ class LIMA_CORELINGUISTICPROCESSINGCLIENT_EXPORT CoreLinguisticProcessingClient
     const std::map<std::string,std::string>& metaData,
     const std::string& pipeline,
     const std::map<std::string, AbstractAnalysisHandler*>& handlers,
-    const std::set<std::string>& inactiveUnits = std::set<std::string>()) const override;
+    const std::set<std::string>& inactiveUnits = std::set<std::string>()
+  ) const override;
 
   std::shared_ptr<AnalysisContent> analyze(
     const std::string& texte,
     const std::map<std::string,std::string>& metaData,
     const std::string& pipeline,
     const std::map<std::string, AbstractAnalysisHandler*>& handlers,
-    const std::set<std::string>& inactiveUnits = std::set<std::string>()) const override
-  ;
+    const std::set<std::string>& inactiveUnits = std::set<std::string>()
+  ) const override;
 };
 
 class CoreLinguisticProcessingClientFactory : public AbstractLinguisticProcessingClientFactory
diff --git a/lima_linguisticprocessing/test/analyzeText.cpp b/lima_linguisticprocessing/test/analyzeText.cpp
index 5ac73d208..a095f3ba6 100644
--- a/lima_linguisticprocessing/test/analyzeText.cpp
+++ b/lima_linguisticprocessing/test/analyzeText.cpp
@@ -25,6 +25,7 @@
 #include "common/QsLog/QsLogCategories.h"
 #include "common/QsLog/QsDebugOutput.h"
 #include "common/AbstractFactoryPattern/AmosePluginsManager.h"
+#include "common/MediaProcessors/MediaProcessors.h"
 
 #include "linguisticProcessing/common/linguisticData/languageData.h"
 #include "linguisticProcessing/client/LinguisticProcessingClientFactory.h"
@@ -52,6 +53,8 @@
 #include <QtCore/QString>
 #include <QtCore/QTimer>
 #include <QtCore/QTextStream>
+#include <QtCore/QJsonDocument>
+#include <QtCore/QJsonObject>
 
 #include "cmd_line_helpers.h"
 
@@ -60,6 +63,7 @@ namespace po = boost::program_options;
 using namespace Lima::LinguisticProcessing;
 using namespace Lima::Common::MediaticData;
 using namespace Lima::Common::Misc;
+using namespace Lima::Common::XMLConfigurationFiles;
 using namespace Lima;
 
 void listunits();
@@ -380,8 +384,8 @@ Options)");
     if (QFileInfo::exists(configDir + "/" + lpConfigFile.c_str()))
     {
       // initialize linguistic processing
-      Lima::Common::XMLConfigurationFiles::XMLConfigurationFileParser lpconfig(
-          (configDir + "/" + lpConfigFile.c_str()));
+      XMLConfigurationFileParser lpconfig(
+        (configDir + "/" + lpConfigFile.c_str()));
       LinguisticProcessingClientFactory::changeable().configureClientFactory(
         clientId,
         lpconfig,
@@ -507,6 +511,7 @@ Options)");
 
     if (splitMode == "lines")
     {
+      std::cerr << "%%%%%%%%%%%%% LINES %%%%%%%%%%%%%%%" << std::endl;
       int lineNum = 0, nbLines = 0;
       std::cerr << "Counting number of lines…";
       while (!qfile.atEnd())
@@ -558,6 +563,7 @@ Options)");
     }
     else if (splitMode == "para")
     {
+      std::cerr << "%%%%%%%%%%%%% PARA %%%%%%%%%%%%%%%" << std::endl;
       auto contentText = QString::fromUtf8(qfile.readAll().constData());
       qfile.close();
       if (contentText.isEmpty())
@@ -642,11 +648,83 @@ Options)");
       TimeUtils::logElapsedTime("ReadInputFile");
       TimeUtils::updateCurrentTime();
 
+    //   // Testing the forging of pipelines. Moved to LimaAnalyzer on client side
+    //   auto langid = Lima::Common::MediaticData::MediaticData::single().getMediaId("ud-eng");
+    //   auto pipe = MediaProcessors::changeable().getPipelineForId(langid, pipeline);
+    //   auto managers = Lima::MediaProcessors::single().managers();
+    //   {
+    //     QString jsonGroupString =
+    //         "{ \"name\":\"cpptftokenizer\", "
+    //         "  \"class\":\"CppUppsalaTensorFlowTokenizer\", "
+    //         "  \"model_prefix\": \"tokenizer-eng\" }";
+    //     auto jsonGroup = QJsonDocument::fromJson(jsonGroupString.toUtf8()).object();
+    //
+    //     GroupConfigurationStructure unitConfig(jsonGroup);
+    //
+    //
+    //     // managers (Manager*) used in init of ProcessUnits are stored in
+    //     // m_pipelineManagers of MediaProcessors
+    //     auto tokenizer = MediaProcessUnit::Factory::getFactory(
+    //       "CppUppsalaTensorFlowTokenizer")->create(
+    //         unitConfig,
+    //         managers[langid]);
+    //     pipe->push_back(tokenizer);
+    //   }
+    //   {
+    //     // <group name="tfmorphosyntax" class="TensorFlowMorphoSyntax">
+    //     //   <param key="model_prefix" value="morphosyntax-eng"/>
+    //     //   <param key="embeddings" value="fasttext-eng.bin"/>
+    //     // </group>
+    //     QString jsonGroupString =
+    //         "{ \"name\":\"tfmorphosyntax\", "
+    //         "  \"class\":\"TensorFlowMorphoSyntax\", "
+    //         "  \"model_prefix\": \"morphosyntax-eng\", "
+    //         "  \"embeddings\": \"fasttext-eng.bin\" "
+    //         "}";
+    //     auto jsonGroup = QJsonDocument::fromJson(jsonGroupString.toUtf8()).object();
+    //     GroupConfigurationStructure unitConfig(jsonGroup);
+    //
+    //     // managers (Manager*) used in init of ProcessUnits are stored in
+    //     // m_pipelineManagers of MediaProcessors
+    //     auto pu = MediaProcessUnit::Factory::getFactory(
+    //       "TensorFlowMorphoSyntax")->create(
+    //         unitConfig,
+    //         managers[langid]);
+    //     pipe->push_back(pu);
+    //   }
+    //   {
+    // // <group name="conllDumper" class="ConllDumper">
+    // //   <param key="handler" value="simpleStreamHandler"/>
+    // //   <param key="fakeDependencyGraph" value="false"/>
+    // // </group>
+    //     QString jsonGroupString =
+    //         "{ \"name\":\"conllDumper\", "
+    //         "  \"class\":\"ConllDumper\", "
+    //         "  \"handler\": \"simpleStreamHandler\" "
+    //         "  \"fakeDependencyGraph\": \"false\", "
+    //         "}";
+    //     auto jsonGroup = QJsonDocument::fromJson(jsonGroupString.toUtf8()).object();
+    //
+    //     GroupConfigurationStructure unitConfig(jsonGroup);
+    //
+    //     // managers (Manager*) used in init of ProcessUnits are stored in
+    //     // m_pipelineManagers of MediaProcessors
+    //     auto pu = MediaProcessUnit::Factory::getFactory(
+    //       "ConllDumper")->create(
+    //         unitConfig,
+    //         managers[langid]);
+    //     pipe->push_back(pu);
+    //   }
+
+
 #ifndef DEBUG_LP
       try {
 #endif
         // analyze it
-        client->analyze(contentText, metaData, pipeline, handlers, inactiveUnits);
+        auto analysis = client->analyze(contentText, metaData, pipeline, handlers, inactiveUnits);
+        // std::cerr << "Running process on newly created process unit" << std::endl;
+        // tokenizer->process(*analysis);
+        // std::cerr << "Done" << std::endl;
 #ifndef DEBUG_LP
       }
       catch (const LinguisticProcessingException& e) {
diff --git a/lima_linguisticprocessing/tests/CMakeLists.txt b/lima_linguisticprocessing/tests/CMakeLists.txt
index 48f94c572..2390e7416 100644
--- a/lima_linguisticprocessing/tests/CMakeLists.txt
+++ b/lima_linguisticprocessing/tests/CMakeLists.txt
@@ -3,87 +3,19 @@
 #
 # SPDX-License-Identifier: MIT
 
-set(test_lima_sources
-    test_lima.cpp
-    lima.cpp
-)
-add_executable(test_lima ${test_lima_sources})
-target_link_libraries(test_lima
-    lima-common-datahandler
-    lima-common-data
-    lima-common-factory
-    lima-common-fsaaccess
-    lima-common-mediaprocessors
-    lima-common-mediaticdata
-    lima-common-misc
-    lima-common-processunitframework
-    lima-common
-    lima-common-time
-    lima-common-tools
-    lima-common-xmlconfigurationfiles
-    lima-linguisticprocessing
-    lima-lp-analysisdict
-    lima-lp-analysisdumpers
-    lima-lp-analysishandlers
-    lima-lp-applyrecognizer
-    lima-lp-automatoncompiler
-    lima-lp-automaton
-    lima-lp-bagofwords
-    lima-lp-client
-    lima-lp-compounds
-    lima-lp-conllureader
-    lima-lp-corefsolver
-    lima-lp-dictionary
-    lima-lp-entitytracker
-    lima-lp-eventanalyzer
-    lima-lp-flattokenizer
-    lima-lp-helpers
-    lima-lp-langdetector
-    lima-lp-lineartextrepresentation
-    lima-lp-linguisticanalysisstructure
-    lima-lp-linguisticdata
-    lima-lp-linguisticprocessing-core
-    lima-lp-linguisticprocessors
-    lima-lp-linguisticresources
-    lima-lp-misc
-    lima-lp-morphologicanalysis
-    lima-lp-postagger
-    lima-lp-propertycode
-    lima-lp-regexmatcher
-    lima-lp-se-datetime
-    lima-lp-semanticanalysis
-    lima-lp-se-number
-    lima-lp-se-person
-    lima-lp-specificentities
-    lima-lp-syntacticanalysis
-    lima-lp-textsegmentation
-    lima-lp-tgv
-    lima-lp-wordsensetagger
-    lima-xp-client
-    lima-xp-core
-    lima-xp-documentsreader
-    lima-xp-qtsgmlentities
-
-)
-
-install(TARGETS test_lima
-  DESTINATION bin
-  COMPONENT runtime)
-
 ########### next test ###############
 
 SET(LimaAnalyzerTest0_SRCS
     LimaAnalyzerTest0.cpp
-    lima.cpp
 )
 
 set (LimaAnalyzerTest0_MOC_HEADERS LimaAnalyzerTest0.h)
 
 add_executable(LimaAnalyzerTest0 ${LimaAnalyzerTest0_SRCS} ${LimaAnalyzerTest0_MOC_OUTFILES})
 target_link_libraries(LimaAnalyzerTest0
-    lima-linguisticprocessing
     Qt${QT_VERSION_MAJOR}::Test
 
+    lima-linguisticprocessing
     lima-common-datahandler
     lima-common-data
     lima-common-factory
@@ -138,6 +70,7 @@ target_link_libraries(LimaAnalyzerTest0
     lima-xp-core
     lima-xp-documentsreader
     lima-xp-qtsgmlentities
+    lima-lp-analyzer
 )
 
 add_test(NAME LimaAnalyzerTest0 COMMAND LimaAnalyzerTest0)
diff --git a/lima_linguisticprocessing/tests/LimaAnalyzerTest0.cpp b/lima_linguisticprocessing/tests/LimaAnalyzerTest0.cpp
index cd74ed90d..3fbdc61c1 100644
--- a/lima_linguisticprocessing/tests/LimaAnalyzerTest0.cpp
+++ b/lima_linguisticprocessing/tests/LimaAnalyzerTest0.cpp
@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: MIT
 
 #include "LimaAnalyzerTest0.h"
-#include "lima.h"
+#include "client/lima.h"
 
 #include "common/AbstractFactoryPattern/AmosePluginsManager.h"
 #include "common/LimaCommon.h"
@@ -58,7 +58,7 @@ void LimaAnalyzerTest0::initTestCase()
 {
 }
 
-void LimaAnalyzerTest0::test_two_instances()
+void LimaAnalyzerTest0::test_two_same_instances()
 {
   // qDebug() << "LimaAnalyzerTest0::test_load";
   {
@@ -73,7 +73,19 @@ void LimaAnalyzerTest0::test_two_instances()
     qDebug() << "LimaAnalyzerTest0::test_load analyzer 2:" << result;
   }
   QVERIFY( true );
+}
+
+void LimaAnalyzerTest0::test_two_different_instances()
+{
+    LimaAnalyzer analyzer0("ud-eng", "deepud", "");
+    auto result1 = analyzer0.analyzeText("Hop ! Hop !");
+    qDebug() << "LimaAnalyzerTest0::test_two_different_instances analyzer 1:" << result1;
+
+    LimaAnalyzer analyzer1("eng", "main", "");
+    auto result2 = analyzer1.analyzeText("One, 2, tree.");
+    qDebug() << "LimaAnalyzerTest0::test_two_different_instances analyzer 2:" << result2;
 
+    QVERIFY( true );
 }
 
 void LimaAnalyzerTest0::test_unknownLanguage() {
@@ -165,11 +177,44 @@ void LimaAnalyzerTest0::test_analyzeText_init_with_lang_and_pipe() {
       typeid(thelima.analyzeText("This is a text on 02/05/2022.")) == typeid(std::string));
 }
 
-void LimaAnalyzerTest0::test_analyzeText_pipeline_not_avail() {
+void LimaAnalyzerTest0::test_analyzeText_pipeline_not_avail()
+{
     LimaAnalyzer lima("ud-eng", "deepud", "");
     lima.analyzeText("This is a text on 02/05/2022.", "other");
     QVERIFY(lima.error);
 }
 
+// void LimaAnalyzerTest0::test_forged_pipeline()
+// {
+//     LimaAnalyzer lima("ud-eng", "empty", "");
+//
+//     // Testing the forging of pipelines
+//     //   auto langid = Lima::Common::MediaticData::MediaticData::single().getMediaId("ud-eng");
+//     //   auto pipe = MediaProcessors::changeable().getPipelineForId(langid, pipeline);
+//     //   auto managers = Lima::MediaProcessors::single().managers();
+//     //   {
+//     std::string jsonGroupString =
+//         "{ \"name\":\"cpptftokenizer\", "
+//         "  \"class\":\"CppUppsalaTensorFlowTokenizer\", "
+//         "  \"model_prefix\": \"tokenizer-eng\" }";
+//     lima.addPipelineUnit("empty", "ud-eng", jsonGroupString);
+//     jsonGroupString =
+//         "{ \"name\":\"tfmorphosyntax\", "
+//         "  \"class\":\"TensorFlowMorphoSyntax\", "
+//         "  \"model_prefix\": \"morphosyntax-eng\", "
+//         "  \"embeddings\": \"fasttext-eng.bin\" "
+//         "}";
+//     lima.addPipelineUnit("empty", "ud-eng", jsonGroupString);
+//     jsonGroupString =
+//         "{ \"name\":\"conllDumper\", "
+//         "  \"class\":\"ConllDumper\", "
+//         "  \"handler\": \"simpleStreamHandler\" "
+//         "  \"fakeDependencyGraph\": \"false\", "
+//         "}";
+//     lima.addPipelineUnit("empty", "ud-eng", jsonGroupString);
+//     lima.analyzeText("This is a text on 02/05/2022.", "ud-eng", "empty");
+//     QVERIFY(lima.error);
+// }
+
 QTEST_GUILESS_MAIN(LimaAnalyzerTest0)
 
diff --git a/lima_linguisticprocessing/tests/LimaAnalyzerTest0.h b/lima_linguisticprocessing/tests/LimaAnalyzerTest0.h
index c9bcdb84a..ec4f8ba19 100644
--- a/lima_linguisticprocessing/tests/LimaAnalyzerTest0.h
+++ b/lima_linguisticprocessing/tests/LimaAnalyzerTest0.h
@@ -16,16 +16,19 @@ class LimaAnalyzerTest0: public QObject
 private Q_SLOTS:
   void initTestCase();
 
-  void test_two_instances();
   void test_unknownLanguage();
   void test_lang_no_prefix();
   void test_several_instances();
+  void test_two_same_instances();
+  void test_two_different_instances();
   void test_analyzeText_lang_not_str();
   void test_analyzeText_lang_not_init();
   void test_analyzeText_pipeline_none_lang_ud();
   void test_analyzeText_pipeline_none_lang_eng();
   void test_analyzeText_init_with_lang_and_pipe();
   void test_analyzeText_pipeline_not_avail();
+  // void test_forged_pipeline();
+
 private:
 };
 
diff --git a/lima_linguisticprocessing/tests/test_lima.cpp b/lima_linguisticprocessing/tests/test_lima.cpp
deleted file mode 100644
index a6d98577b..000000000
--- a/lima_linguisticprocessing/tests/test_lima.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2019-2022 CEA LIST
-// SPDX-FileCopyrightText: 2019-2022 CEA LIST <gael.de-chalendar@cea.fr>
-//
-// SPDX-License-Identifier: MIT
-
-/****************************************************************************
-**
-** Copyright (C) 2018 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the Qt for Python examples of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:BSD$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** BSD License Usage
-** Alternatively, you may use this file under the terms of the BSD license
-** as follows:
-**
-** "Redistribution and use in source and binary forms, with or without
-** modification, are permitted provided that the following conditions are
-** met:
-**   * Redistributions of source code must retain the above copyright
-**     notice, this list of conditions and the following disclaimer.
-**   * Redistributions in binary form must reproduce the above copyright
-**     notice, this list of conditions and the following disclaimer in
-**     the documentation and/or other materials provided with the
-**     distribution.
-**   * Neither the name of The Qt Company Ltd nor the names of its
-**     contributors may be used to endorse or promote products derived
-**     from this software without specific prior written permission.
-**
-**
-** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-
-#include "lima.h"
-
-#include <iostream>
-
-int main(int /*argc*/, char** /*argv[]*/)
-{
-  LimaAnalyzer analyzer0("ud-eng", "deepud", "");
-  std::cerr << analyzer0.analyzeText("Hop ! Hop !") << std::endl;
-
-  LimaAnalyzer analyzer1("eng", "main", "");
-  std::cerr << analyzer1.analyzeText("One, 2, tree.") << std::endl;
-}