ojob.io/raid/logParser: fix extra field

raid/logParser.json

@@ -1 +1 @@
-{"help":{"text":"Reads a ndjson logs file and converts it to readable human logs (version 20250204a)","expects":[{"name":"file","desc":"The file path to the ndjson log files (either .ndjson or .ndjson.gz)","example":"logs_12345.ndjson.gz","mandatory":false},{"name":"folder","desc":"The folder path to the ndjson log files (either .ndjson or .ndjson.gz). Will convert one by one into .log files.","example":"/some/path","mandatory":false},{"name":"filter","desc":"Comma separated pairs of field value filters (for example: \"field1:abc,field2:xyz\")","example":"field1:abc,field2:xyz","mandatory":false},{"name":"notfilter","desc":"Comma separated pairs of field value not filters (for example: \"field1:abc,field2:xyz\")","example":"field1:abc,field2:xyz","mandatory":false},{"name":"type","desc":"If you want to specify any longer not included by default (for example: AuditLogger)","example":"AuditLogger","mandatory":false},{"name":"icase","desc":"If icase=false then filters will be case sensitive (default: icase=true)","example":"false","mandatory":false},{"name":"inception","desc":"If inception=true it will handle cases where JSON is included in the message field (default: inception=true)","example":"true","mandatory":false},{"name":"injson","desc":"Boolean to determine if output should be in JSON format","example":"true","mandatory":false},{"name":"alltypes","desc":"Boolean to determine if all types should be output","example":"true","mandatory":false}]},"init":{"templateAudit":"{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{code}} {{logger_name}} - {{clientType}}, {{clientVersion}}, {{clientUsername}}, {{remoteIP}}, {{remoteHostname}}, {{clientIP}}, {{clientHostname}}, {{referrer}}, {{username}}, {{sessionId}}, {{activityType}}, {{activityName}}, {{{activityDescr}}}, {{{result}}}, {{errorCode}}, {{{failureDescr}}}, {{operationTime}} {{stack_trace}}","templateRegular":"{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{code}} {{logger_name}} - {{{message}}} {{{stack_trace}}}","templateDefault":"{{_time}} | {{level }} | {{{message}}}"},"todo":["Init","Main process"],"ojob":{"sequential":true,"opacks":[{"openaf":20220510}],"catch":"printErr(\"ERROR: \" + exception)\n","logToConsole":false},"jobs":[{"name":"Init","exec":"ow.loadTemplate();ow.template.addFormatHelpers();ow.template.addOpenAFHelpers();ow.template.addConditionalHelpers();global.wildcardTest=function(b,a,c){c=_$(c).isBoolean().default(!1);a=\"^\"+a.replace(/[.+^${}()|[\\]\\\\]/g,\"\\\\$1\").replace(/\\*/g,\".*\").replace(/\\?/g,\".\")+\"$\";return(new RegExp(a,c?__:\"i\")).test(b)};global._p=function(b,a){return isDef(a)?ioStreamWrite(a,b+\"\\n\"):print(b)}"},{"name":"Parse ndjson","deps":"Init","catch":"sprintErr(exception)","check":{"in":{"file":"isString.default(__)","output":"isString.default(__)","filter":"isString.default(__)","notfilter":"isString.default(__)","type":"isString.default(__)","icase":"toBoolean.isBoolean.default(true)","inception":"toBoolean.isBoolean.default(true)","injson":"toBoolean.isBoolean.default(false)","alltypes":"toBoolean.isBoolean.default(false)"}},"exec":"var g=__,k=__,d={auditLines:0,regularLines:0,defaultLines:0,origLines:0,filteredLines:0,convertedLines:0,nonJson:0,multiLine:0};if(isDef(args.file)&&(args.file.endsWith(\".ndjson\")&&(g=io.readFileStream(args.file)),args.file.endsWith(\".ndjson.gz\")&&(g=io.readFileGzipStream(args.file)),isUnDef(g)))throw\"Can't recognize file '\"+args.file+\"' as a ndjson or ndjson.gz\";var m=ow.template.execCompiled(ow.template.compile(args.init.templateAudit)),n=ow.template.execCompiled(ow.template.compile(args.init.templateRegular)),\np=ow.template.execCompiled(ow.template.compile(args.init.templateDefault));isDef(args.output)&&(k=args.output.endsWith(\".gz\")?io.writeFileGzipStream(args.output):io.writeFileStream(args.output));args.injson&&isUnDef(args.__format)&&(args.__format=\"json\");var e=__,l=function(c,h){c=String(c);d.origLines++;isUnDef(e)&&0!=c.length&&c.trim().startsWith(\"{\")&&(e=\"\");if(isDef(e)){if(0!=c.length&&!c.trim().endsWith(\"}\")){e+=c.trim();d.multiLine++;return}0<e.length&&(c=e+c,e=__)}if(0==c.length||0<c.length&&\n\"{\"!=c.trim().substring(0,1))isUnDef(args.filter)&&isUnDef(args.notfilter)&&isUnDef(args.type)&&!args.injson&&global._p(c,h),e=__,d.nonJson++;else{var f=!0;try{var a=jsonParse(c);args.inception&&isMap(a)&&isString(a.message)&&a.message.startsWith(\"{\")&&a.message.endsWith(\"}\")&&(a=jsonParse(a.message))}catch(b){sprintErr(b)}if(isMap(a))if(isString(args.filter)&&(0<=args.filter.indexOf(\"*\")||0<=args.filter.indexOf(\"?\")?args.filter.split(\",\").forEach(function(b){b=b.split(\":\");if(isUnDef(a[b[0]])||!global.wildcardTest(a[b[0]],\nb[1],!args.icase))f=!1}):args.filter.split(\",\").forEach(function(b){b=b.split(\":\");if(isUnDef(a[b[0]])||String(a[b[0]])!=String(b[1]))f=!1})),isString(args.notfilter)&&(0<=args.notfilter.indexOf(\"*\")||0<=args.notfilter.indexOf(\"?\")?args.notfilter.split(\",\").forEach(function(b){b=b.split(\":\");isDef(a[b[0]])&&global.wildcardTest(a[b[0]],b[1],!args.icase)&&(f=!1)}):args.notfilter.split(\",\").forEach(function(b){b=b.split(\":\");isDef(a[b[0]])&&String(a[b[0]])==String(b[1])&&(f=!1)})),isDef(args.type)&&\na.logger_name!=args.type&&(f=!1),isUnDef(args.type)&&\"AuditLogger\"==a.logger_name&&(d.auditLines++,f=!1),args.alltypes&&(f=!0),f)switch(c=function(b){b=isDef(b[\"@timestamp\"])?b[\"@timestamp\"]:b.parse_timestamp;b.endsWith(\"Z\")&&0<b.indexOf(\".\")&&24>b.length&&(b=[b.substring(0,19),\".\",b.substring(20,b.length-1),\"0\".repeat(24-b.length),\"Z\"].join(\"\"));0<b.indexOf(\"+\")&&(b=[b.substring(0,23),\"Z\"].join(\"\"));b.match(/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}\\d+Z$/)&&(b=[b.substring(0,23),\"Z\"].join(\"\"));\nreturn b?ow.format.fromDate(new Date(b),\"yyyy-MM-dd HH:mm:ss,SSS\"):b},isString(a[\"@timestamp\"])&&0<a[\"@timestamp\"].indexOf(\"T\")&&0<a[\"@timestamp\"].indexOf(\".\")&&a[\"@timestamp\"].endsWith(\"Z\")&&(a[\"@timestamp\"]=ow.format.toDate(a[\"@timestamp\"],\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\")),a.logger_name){case \"AuditLogger\":a._time=c(a);a.thread_name=a.thread_name.substr(-20);args.injson?ow.oJob.output(a,args):global._p(m(a),h);d.auditLines++;d.convertedLines++;break;default:try{isUnDef(a.code)||isUnDef(a.thread_name)?\n(a._time=c(a),isDef(a.thread_name)&&(a.message=\"[\"+a.thread_name+\"] \"+a.message),args.injson?ow.oJob.output(a,args):global._p(p(a),h),d.defaultLines++):(a._time=c(a),a.thread_name=a.thread_name.substr(-20),args.injson?ow.oJob.output(a,args):global._p(n(a),h),d.regularLines++),d.convertedLines++}catch(b){printErr(\"ERROR | \"+b+\" | record = \"+af.toSLON(a))}}else d.filteredLines++,args.alltypes&&(isMap(a)?print(af.toSLON(a)):print(a));else isUnDef(args.filter)&&isUnDef(args.notfilter)&&isUnDef(args.type)&&\n!args.injson&&global._p(a,h)}};isDef(args.file)?(ioStreamReadLines(g,function(c){return l(c,k)},\"\\n\"),g.close(),isDef(k)&&k.close(),$set(\"metrics_\"+args.file,d)):io.pipeLn(l)"},{"name":"Main process","deps":"Init","check":{"in":{"file":"isString.default(__)","folder":"isString.default(__)"}},"exec":"isDef(args.folder)?io.fileExists(args.folder)?(tlog(\"Finding .ndjson.gz and .ndjson files in '{{folder}}'...\",args),$ch(\"data\").create(),$from(listFilesRecursive(args.folder)).equals(\"isFile\",!0).ends(\"filename\",\".ndjson\").orEnds(\"filename\",\".ndjson.gz\").pselect(function(a){a.target=a.filepath.replace(/\\.ndjson(.gz)?$/,\".log$1\");a.sizeAbbr=ow.format.toBytesAbbreviation(a.size);tlog(\"Converting {{filepath}} ({{sizeAbbr}}) to {{target}}...\",a);$job(\"Parse ndjson\",merge(args,{output:a.target,\nfile:a.filepath}));var c=io.fileInfo(a.target);c.sizeAbbr=ow.format.toBytesAbbreviation(c.size);var b=$get(\"metrics_\"+a.filepath);$ch(\"data\").set({canonicalPath:a.canonicalPath},{source:a.filepath,sourceSize:a.sizeAbbr,\"#source\":b.origLines,target:a.target,targeSize:c.sizeAbbr,\"#filtered\":b.filteredLines,\"#nonJson\":b.nonJson,\"#multiLine\":b.multiLine,\"#target\":b.convertedLines});tlog(\"Converted {{filepath}}\",a)}),$doWait($doAll([])),args.__format=_$(args.__format).default(\"table\"),print(\"\"),ow.oJob.output($from($ch(\"data\").getAll()).sort(\"source\").select(),\nargs)):tlogErr(\"Folder '{{folder}}' not found.\",args):$job(\"Parse ndjson\",args)"}]}
+{"help":{"text":"Reads a ndjson logs file and converts it to readable human logs (version 20250204a)","expects":[{"name":"file","desc":"The file path to the ndjson log files (either .ndjson or .ndjson.gz)","example":"logs_12345.ndjson.gz","mandatory":false},{"name":"folder","desc":"The folder path to the ndjson log files (either .ndjson or .ndjson.gz). Will convert one by one into .log files.","example":"/some/path","mandatory":false},{"name":"filter","desc":"Comma separated pairs of field value filters (for example: \"field1:abc,field2:xyz\")","example":"field1:abc,field2:xyz","mandatory":false},{"name":"notfilter","desc":"Comma separated pairs of field value not filters (for example: \"field1:abc,field2:xyz\")","example":"field1:abc,field2:xyz","mandatory":false},{"name":"type","desc":"If you want to specify any longer not included by default (for example: AuditLogger)","example":"AuditLogger","mandatory":false},{"name":"icase","desc":"If icase=false then filters will be case sensitive (default: icase=true)","example":"false","mandatory":false},{"name":"inception","desc":"If inception=true it will handle cases where JSON is included in the message field (default: inception=true)","example":"true","mandatory":false},{"name":"injson","desc":"Boolean to determine if output should be in JSON format","example":"true","mandatory":false},{"name":"alltypes","desc":"Boolean to determine if all types should be output","example":"true","mandatory":false}]},"init":{"templateAudit":"{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{logger_name}} {{code}} - {{clientType}}, {{clientVersion}}, {{clientUsername}}, {{remoteIP}}, {{remoteHostname}}, {{clientIP}}, {{clientHostname}}, {{referrer}}, {{username}}, {{sessionId}}, {{activityType}}, {{activityName}}, {{{activityDescr}}}, {{{result}}}, {{errorCode}}, {{{failureDescr}}}, {{operationTime}} {{stack_trace}}","templateRegular":"{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{logger_name}} {{code}} [{{executionId}}] - {{{message}}} {{{stack_trace}}}","templateDefault":"{{_time}} | {{level }} | {{{message}}}"},"todo":["Init","Main process"],"ojob":{"sequential":true,"opacks":[{"openaf":20220510}],"catch":"printErr(\"ERROR: \" + exception)\n","logToConsole":false},"jobs":[{"name":"Init","exec":"ow.loadTemplate();ow.template.addFormatHelpers();ow.template.addOpenAFHelpers();ow.template.addConditionalHelpers();global.wildcardTest=function(b,a,c){c=_$(c).isBoolean().default(!1);a=\"^\"+a.replace(/[.+^${}()|[\\]\\\\]/g,\"\\\\$1\").replace(/\\*/g,\".*\").replace(/\\?/g,\".\")+\"$\";return(new RegExp(a,c?__:\"i\")).test(b)};global._p=function(b,a){return isDef(a)?ioStreamWrite(a,b+\"\\n\"):print(b)}"},{"name":"Parse ndjson","deps":"Init","catch":"sprintErr(exception)","check":{"in":{"file":"isString.default(__)","output":"isString.default(__)","filter":"isString.default(__)","notfilter":"isString.default(__)","type":"isString.default(__)","icase":"toBoolean.isBoolean.default(true)","inception":"toBoolean.isBoolean.default(true)","injson":"toBoolean.isBoolean.default(false)","alltypes":"toBoolean.isBoolean.default(false)"}},"exec":"var g=__,k=__,d={auditLines:0,regularLines:0,defaultLines:0,origLines:0,filteredLines:0,convertedLines:0,nonJson:0,multiLine:0};if(isDef(args.file)&&(args.file.endsWith(\".ndjson\")&&(g=io.readFileStream(args.file)),args.file.endsWith(\".ndjson.gz\")&&(g=io.readFileGzipStream(args.file)),isUnDef(g)))throw\"Can't recognize file '\"+args.file+\"' as a ndjson or ndjson.gz\";var m=ow.template.execCompiled(ow.template.compile(args.init.templateAudit)),n=ow.template.execCompiled(ow.template.compile(args.init.templateRegular)),\np=ow.template.execCompiled(ow.template.compile(args.init.templateDefault));isDef(args.output)&&(k=args.output.endsWith(\".gz\")?io.writeFileGzipStream(args.output):io.writeFileStream(args.output));args.injson&&isUnDef(args.__format)&&(args.__format=\"json\");var e=__,l=function(c,h){c=String(c);d.origLines++;isUnDef(e)&&0!=c.length&&c.trim().startsWith(\"{\")&&(e=\"\");if(isDef(e)){if(0!=c.length&&!c.trim().endsWith(\"}\")){e+=c.trim();d.multiLine++;return}0<e.length&&(c=e+c,e=__)}if(0==c.length||0<c.length&&\n\"{\"!=c.trim().substring(0,1))isUnDef(args.filter)&&isUnDef(args.notfilter)&&isUnDef(args.type)&&!args.injson&&global._p(c,h),e=__,d.nonJson++;else{var f=!0;try{var a=jsonParse(c);args.inception&&isMap(a)&&isString(a.message)&&a.message.startsWith(\"{\")&&a.message.endsWith(\"}\")&&(a=jsonParse(a.message))}catch(b){sprintErr(b)}if(isMap(a))if(isString(args.filter)&&(0<=args.filter.indexOf(\"*\")||0<=args.filter.indexOf(\"?\")?args.filter.split(\",\").forEach(function(b){b=b.split(\":\");if(isUnDef(a[b[0]])||!global.wildcardTest(a[b[0]],\nb[1],!args.icase))f=!1}):args.filter.split(\",\").forEach(function(b){b=b.split(\":\");if(isUnDef(a[b[0]])||String(a[b[0]])!=String(b[1]))f=!1})),isString(args.notfilter)&&(0<=args.notfilter.indexOf(\"*\")||0<=args.notfilter.indexOf(\"?\")?args.notfilter.split(\",\").forEach(function(b){b=b.split(\":\");isDef(a[b[0]])&&global.wildcardTest(a[b[0]],b[1],!args.icase)&&(f=!1)}):args.notfilter.split(\",\").forEach(function(b){b=b.split(\":\");isDef(a[b[0]])&&String(a[b[0]])==String(b[1])&&(f=!1)})),isDef(args.type)&&\na.logger_name!=args.type&&(f=!1),isUnDef(args.type)&&\"AuditLogger\"==a.logger_name&&(d.auditLines++,f=!1),args.alltypes&&(f=!0),f)switch(c=function(b){b=isDef(b[\"@timestamp\"])?b[\"@timestamp\"]:b.parse_timestamp;b.endsWith(\"Z\")&&0<b.indexOf(\".\")&&24>b.length&&(b=[b.substring(0,19),\".\",b.substring(20,b.length-1),\"0\".repeat(24-b.length),\"Z\"].join(\"\"));0<b.indexOf(\"+\")&&(b=[b.substring(0,23),\"Z\"].join(\"\"));b.match(/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}\\d+Z$/)&&(b=[b.substring(0,23),\"Z\"].join(\"\"));\nreturn b?ow.format.fromDate(new Date(b),\"yyyy-MM-dd HH:mm:ss,SSS\"):b},isString(a[\"@timestamp\"])&&0<a[\"@timestamp\"].indexOf(\"T\")&&0<a[\"@timestamp\"].indexOf(\".\")&&a[\"@timestamp\"].endsWith(\"Z\")&&(a[\"@timestamp\"]=ow.format.toDate(a[\"@timestamp\"],\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\")),a.logger_name){case \"AuditLogger\":a._time=c(a);a.thread_name=a.thread_name.substr(-20);args.injson?ow.oJob.output(a,args):global._p(m(a),h);d.auditLines++;d.convertedLines++;break;default:try{isUnDef(a.code)||isUnDef(a.thread_name)?\n(a._time=c(a),isDef(a.thread_name)&&(a.message=\"[\"+a.thread_name+\"] \"+a.message),args.injson?ow.oJob.output(a,args):global._p(p(a),h),d.defaultLines++):(a._time=c(a),a.thread_name=a.thread_name.substr(-20),args.injson?ow.oJob.output(a,args):global._p(n(a),h),d.regularLines++),d.convertedLines++}catch(b){printErr(\"ERROR | \"+b+\" | record = \"+af.toSLON(a))}}else d.filteredLines++,args.alltypes&&(isMap(a)?print(af.toSLON(a)):print(a));else isUnDef(args.filter)&&isUnDef(args.notfilter)&&isUnDef(args.type)&&\n!args.injson&&global._p(a,h)}};isDef(args.file)?(ioStreamReadLines(g,function(c){return l(c,k)},\"\\n\"),g.close(),isDef(k)&&k.close(),$set(\"metrics_\"+args.file,d)):io.pipeLn(l)"},{"name":"Main process","deps":"Init","check":{"in":{"file":"isString.default(__)","folder":"isString.default(__)"}},"exec":"isDef(args.folder)?io.fileExists(args.folder)?(tlog(\"Finding .ndjson.gz and .ndjson files in '{{folder}}'...\",args),$ch(\"data\").create(),$from(listFilesRecursive(args.folder)).equals(\"isFile\",!0).ends(\"filename\",\".ndjson\").orEnds(\"filename\",\".ndjson.gz\").pselect(function(a){a.target=a.filepath.replace(/\\.ndjson(.gz)?$/,\".log$1\");a.sizeAbbr=ow.format.toBytesAbbreviation(a.size);tlog(\"Converting {{filepath}} ({{sizeAbbr}}) to {{target}}...\",a);$job(\"Parse ndjson\",merge(args,{output:a.target,\nfile:a.filepath}));var c=io.fileInfo(a.target);c.sizeAbbr=ow.format.toBytesAbbreviation(c.size);var b=$get(\"metrics_\"+a.filepath);$ch(\"data\").set({canonicalPath:a.canonicalPath},{source:a.filepath,sourceSize:a.sizeAbbr,\"#source\":b.origLines,target:a.target,targeSize:c.sizeAbbr,\"#filtered\":b.filteredLines,\"#nonJson\":b.nonJson,\"#multiLine\":b.multiLine,\"#target\":b.convertedLines});tlog(\"Converted {{filepath}}\",a)}),$doWait($doAll([])),args.__format=_$(args.__format).default(\"table\"),print(\"\"),ow.oJob.output($from($ch(\"data\").getAll()).sort(\"source\").select(),\nargs)):tlogErr(\"Folder '{{folder}}' not found.\",args):$job(\"Parse ndjson\",args)"}]}

raid/logParser.yaml

@@ -40,8 +40,8 @@ help:
     mandatory: false
 
 init:
-  templateAudit  : "{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{code}} {{logger_name}} - {{clientType}}, {{clientVersion}}, {{clientUsername}}, {{remoteIP}}, {{remoteHostname}}, {{clientIP}}, {{clientHostname}}, {{referrer}}, {{username}}, {{sessionId}}, {{activityType}}, {{activityName}}, {{{activityDescr}}}, {{{result}}}, {{errorCode}}, {{{failureDescr}}}, {{operationTime}} {{stack_trace}}"
-  templateRegular: "{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{code}} {{logger_name}} - {{{message}}} {{{stack_trace}}}"
+  templateAudit  : "{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{logger_name}} {{code}} - {{clientType}}, {{clientVersion}}, {{clientUsername}}, {{remoteIP}}, {{remoteHostname}}, {{clientIP}}, {{clientHostname}}, {{referrer}}, {{username}}, {{sessionId}}, {{activityType}}, {{activityName}}, {{{activityDescr}}}, {{{result}}}, {{errorCode}}, {{{failureDescr}}}, {{operationTime}} {{stack_trace}}"
+  templateRegular: "{{_time}} [{{$f '%-20.20s' thread_name}}] {{$f '%-5s' level}} {{logger_name}} {{code}} [{{executionId}}] - {{{message}}} {{{stack_trace}}}"
   templateDefault: "{{_time}} | {{level }} | {{{message}}}"
 
 todo:

raid/logParser0.yaml

@@ -148,7 +148,7 @@ jobs:
       }
       if (args.injson) {
         if (!isMap(data))
-          data = { message: data }
+          data = { message: data }
         else
           if (Object.keys(data).length == 0) return
       }