HPCC-31775 Hook jlog to use api to log operator messages

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.com>

dali/base/daclient.cpp

@@ -29,6 +29,7 @@
 #include "dautils.hpp"
 
 #include "daclient.hpp"
+#include "sysinfologger.hpp"
 
 extern bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned timeout,DaliClientRole role);
 extern void stopClientProcess();
@@ -141,6 +142,8 @@ bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned mpport,
     covengrp->Release();
     queryLogMsgManager()->setSession(myProcessSession());
 
+    if (getGlobalConfigSP()->getPropBool("@enableGlobalSysLog", true))
+        UseDaliForOperatorMessages();
     if (!isContainerized()) // The Environment is bare-metal only
     {
         // auto install environment monitor for server roles
@@ -177,6 +180,7 @@ void removeShutdownHook(IDaliClientShutdown &shutdown)
 
 void closedownClientProcess()
 {
+    UseDaliForOperatorMessages(false);
     if (!daliClientIsActive)
         return;
     while (shutdownHooks.ordinality())

dali/base/sysinfologger.cpp

@@ -694,3 +694,79 @@ unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigne
 
     return count;
 }
+
+class DaliMsgLoggerHandler : public CInterfaceOf<ILogMsgHandler>
+{
+public:
+    DaliMsgLoggerHandler(unsigned _messageFields=MSGFIELD_all) : messageFields(_messageFields)
+    {
+    }
+    virtual void handleMessage(const LogMsg & msg) override
+    {
+        LogMsgSysInfo sysInfo = msg.querySysInfo();
+        time_t timeNum = sysInfo.queryTime();
+        unsigned __int64 ts = sysInfo.queryTime() * 1000000 + sysInfo.queryUSecs();
+        logSysInfoError(msg.queryCategory(), msg.queryCode(), queryComponentName(), msg.queryText(), ts);
+    }
+    virtual bool needsPrep() const  override
+    {
+        return false;
+    }
+    virtual void prep() override
+    {
+    }
+    virtual unsigned queryMessageFields() const override
+    {
+        return messageFields;
+    }
+    virtual void setMessageFields(unsigned _fields = MSGFIELD_all) override
+    {
+        messageFields = _fields;
+    }
+    virtual void addToPTree(IPropertyTree * parent) const override
+    {
+        IPropertyTree * handlerTree = createPTree(ipt_caseInsensitive);
+        handlerTree->setProp("@type", "globalmessages");
+        handlerTree->setPropInt("@fields", messageFields);
+        parent->addPropTree("handler", handlerTree);
+    }
+    virtual int flush() override
+    {
+        return 0;
+    }
+    virtual bool getLogName(StringBuffer &name) const override
+    {
+        return false;
+    }
+    virtual offset_t getLogPosition(StringBuffer &logFileName) const override
+    {
+        return 0;
+    }
+private:
+    unsigned messageFields = MSGFIELD_all;
+};
+
+void UseDaliForOperatorMessages(bool use)
+{
+    static ILogMsgHandler *msgHandler=NULL;
+    if (use==(msgHandler!=NULL))
+        return;
+    if (use)
+    {
+        msgHandler = getDaliMsgLoggerHandler();
+        ILogMsgFilter * operatorFilter = getCategoryLogMsgFilter(MSGAUD_operator,
+                                                                 MSGCLS_disaster|MSGCLS_error|MSGCLS_warning,
+                                                                 WarnMsgThreshold);
+        queryLogMsgManager()->addMonitorOwn(msgHandler, operatorFilter);
+    }
+    else
+    {
+        queryLogMsgManager()->removeMonitor(msgHandler);
+        msgHandler = NULL;
+    }
+}
+
+ILogMsgHandler * getDaliMsgLoggerHandler()
+{
+    return new DaliMsgLoggerHandler();
+}

dali/base/sysinfologger.hpp

@@ -78,7 +78,7 @@ typedef IIteratorOf<ISysInfoLoggerMsg> ISysInfoLoggerMsgIterator;
 SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source=nullptr);
 SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source=nullptr);
 SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *source=nullptr);
-SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(ISysInfoLoggerMsgFilter * msgFilter);
+SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter);
 
 SYSINFO_API unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts);
 SYSINFO_API unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter);
@@ -89,4 +89,6 @@ SYSINFO_API unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilte
 SYSINFO_API bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr);
 SYSINFO_API unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source=nullptr);
 
+SYSINFO_API ILogMsgHandler * getDaliMsgLoggerHandler();
+SYSINFO_API void UseDaliForOperatorMessages(bool use=true);
 #endif

esp/scm/ws_workunits.ecm

@@ -109,7 +109,7 @@ ESPservice [
     ESPmethod [cache_seconds(60), min_ver("1.57")] WUListArchiveFiles(WUListArchiveFilesRequest, WUListArchiveFilesResponse);
     ESPmethod [cache_seconds(60), min_ver("1.57")] WUGetArchiveFile(WUGetArchiveFileRequest, WUGetArchiveFileResponse);
     ESPmethod [cache_seconds(60), min_ver("1.61")] WUGetNumFileToCopy(WUGetNumFileToCopyRequest, WUGetNumFileToCopyResponse);
-    ESPmethod [min_ver("1.71")] WUDetails(WUDetailsRequest, WUDetailsResponse);
+    ESPmethod [min_ver("1.72")] WUDetails(WUDetailsRequest, WUDetailsResponse);
     ESPmethod [cache_seconds(600), min_ver("1.71")] WUDetailsMeta(WUDetailsMetaRequest, WUDetailsMetaResponse);
     ESPmethod [clear_cache_group, min_ver("1.72")] WUEclDefinitionAction(WUEclDefinitionActionRequest, WUEclDefinitionActionResponse);
     ESPmethod [cache_seconds(60), min_ver("1.74")] WUGetThorJobQueue(WUGetThorJobQueueRequest, WUGetThorJobQueueResponse);

esp/scm/ws_workunits_struct.ecm

@@ -836,6 +836,7 @@ ESPstruct [nil_remove] WUResponseNote
     nonNegativeInteger ErrorCode;
     string Severity;
     nonNegativeInteger Cost;
+    [min_ver("1.72")] uint64 Id; // unique identifier for note
 };
 
 ESPstruct [nil_remove] WUResponseScope

esp/services/ws_workunits/ws_wudetails.cpp

@@ -22,6 +22,9 @@
 #include "workunit.hpp"
 #include "jset.hpp"
 #include "jstatcodes.h"
+#include "sysinfologger.hpp"
+
+#define COMPONENT_NOTE_MASK 0xFFFFFC0
 
 typedef std::pair<WuAttr, StringBuffer> AttribValuePair;
 bool operator==(const AttribValuePair & p1, const AttribValuePair & p2)
@@ -487,8 +490,13 @@ void WUDetails::processRequest(IEspWUDetailsRequest &req, IEspWUDetailsResponse
         {
             Owned<IPropertyTree> componentConfig = getComponentConfig();
             IArrayOf<IEspWUResponseNote> espWuResponseNotes;
-
+            // Global messages
+            // Note: there are 2 types of 'notes':
+            // 1) static component warnings (usual from Helm templates)
+            // 2) dynamic global messages generated from executing components
+            // Static component messages will have COMPONENT_NOTE_MASK.
             Owned<IPropertyTreeIterator> iter = componentConfig->getElements("warnings");
+            unsigned __int64 id=1;
             ForEach(*iter)
             {
                 IPropertyTree & cur = iter->query();
@@ -502,6 +510,27 @@ void WUDetails::processRequest(IEspWUDetailsRequest &req, IEspWUDetailsResponse
                     espWuResponseNote->setErrorCode_null();
                 espWuResponseNote->setSeverity(cur.queryProp("@severity"));
                 espWuResponseNote->setCost(0);
+                VStringBuffer idstr("%" I64F "u", id|COMPONENT_NOTE_MASK);
+                espWuResponseNote->setId(idstr.str());
+                espWuResponseNotes.append(*espWuResponseNote.getClear());
+                id++;
+            }
+            Owned<ISysInfoLoggerMsgFilter> msgFilter = createSysInfoLoggerMsgFilter();
+            msgFilter->setVisibleOnly();
+            Owned<ISysInfoLoggerMsgIterator> msgIter = createSysInfoLoggerMsgIterator(msgFilter);
+            ForEach(*msgIter)
+            {
+                ISysInfoLoggerMsg & sysInfoMsg = msgIter->query();
+                Owned<IEspWUResponseNote> espWuResponseNote = createWUResponseNote("","");
+                StringBuffer tmpbuf;
+                encodeXML(sysInfoMsg.queryMsg(), tmpbuf, ENCODE_NEWLINES, strlen(sysInfoMsg.queryMsg()), true);
+                espWuResponseNote->setSource(sysInfoMsg.querySource());
+                espWuResponseNote->setMessage(tmpbuf.str());
+                espWuResponseNote->setErrorCode(sysInfoMsg.queryLogMsgCode());
+                espWuResponseNote->setSeverity(LogMsgClassToVarString(sysInfoMsg.queryClass()));
+                espWuResponseNote->setCost(0);
+                VStringBuffer idstr("%" I64F "u", sysInfoMsg.queryLogMsgId());
+                espWuResponseNote->setId(idstr.str());
                 espWuResponseNotes.append(*espWuResponseNote.getClear());
             }
             Owned<IEspWUResponseScope> respScope = createWUResponseScope("","");

testing/unittests/dalitests.cpp

@@ -3182,6 +3182,7 @@ class DaliSysInfoLoggerTester : public CppUnit::TestFixture
     void testInit()
     {
         daliClientInit();
+        UseDaliForOperatorMessages(false);
     }
     void testWrite()
     {