fix(logstash-http): clean unnecessary dependencies and fix data serialization/mapping

Author: Romakita

.yarn/cache/@types-axios-npm-0.14.0-42f41917ea-12a230b940.zip

@@ -17,6 +17,7 @@ npm install --save @tsed/logger-logstash-http
 - `options.logChannel` - `string` (optional) - also used to identify your application's logs [but in a more specific way]
 - `options.logType` - `string` (optional) - used for the `type` field in the logstash data
 - `options.timeout` - `integer` (optional, defaults to 5000ms) - the timeout for the HTTP request.
+- `options.delayToFlush` - `integer` (optional, defaults to 0) - the delay before flushing buffer if the max buffer isn't reached.
 - `options.maxBuffer` - `integer` (optional, defaults to 0) - Group bulk request by the maxBuffer number. By Default the buffer is disabled.
 - `options.retriesOptions` - `object` (optional) - Configure retries strategy. See [axios-retry](https://www.google.com/search?client=firefox-b-d&q=axios-retry) options for more details.
 

.yarn/cache/axios-npm-0.21.1-d192f6b3b3-c87915fa0b.zip

@@ -113,4 +113,4 @@
     ]
   },
   "packageManager": "yarn@3.2.1"
-}
+}

.yarn/cache/axios-npm-0.26.1-a6641ce4e3-d9eb58ff4b.zip

@@ -47,4 +47,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

docs/appenders/logstash-http.md

@@ -48,4 +48,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

package.json

@@ -48,4 +48,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/connect/package.json

@@ -48,4 +48,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/file/package.json

@@ -45,6 +45,5 @@
   },
   "devDependencies": {
     "typescript": "4.7.4"
-  },
-  "peerDependencies": {}
-}
+  }
+}

packages/insight/package.json

@@ -49,4 +49,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/logentries/package.json

@@ -40,8 +40,7 @@
   },
   "homepage": "https://github.com/tsedio/logger",
   "dependencies": {
-    "@types/axios": "^0.14.0",
-    "axios": "^0.26.1",
+    "axios": "0.27.2",
     "axios-retry": "^3.3.1",
     "tslib": "2.3.1"
   },
@@ -51,4 +50,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/logger/package.json

@@ -29,6 +29,7 @@ export class LogStashHttpOptions {
   logChannel: string;
   auth?: AxiosBasicCredentials;
   timeout?: number;
+  delayToFlush?: number;
   params?: Record<string, any>;
   headers?: Record<string, any>;
   retryOptions?: IAxiosRetryConfig;
@@ -39,6 +40,7 @@ export class LogStashHttpAppender extends BaseAppender<LogStashHttpOptions> {
   private client: ReturnType<typeof axios.create>;
 
   #buffer: Record<string, any>[] = [];
+  #timer: NodeJS.Timeout;
 
   build() {
     if ($log.level !== "OFF" && this.config.options) {
@@ -48,10 +50,8 @@ export class LogStashHttpAppender extends BaseAppender<LogStashHttpOptions> {
         timeout: this.config.options.timeout || 5000,
         params: this.config.options.params,
         headers: {
-          ...(this.config.options.headers || {}),
-          "Content-Type": "application/x-ndjson"
-        },
-        withCredentials: true
+          ...(this.config.options.headers || {})
+        }
       });
 
       axiosRetry(this.client, {
@@ -67,67 +67,83 @@ export class LogStashHttpAppender extends BaseAppender<LogStashHttpOptions> {
     const level = loggingEvent.level.toString().toLowerCase();
 
     if (level !== "off") {
-      const logstashEvent = [
-        {
-          ...loggingEvent.getData(),
-          message: format(loggingEvent.getMessage()),
-          context: loggingEvent.context.toJSON(),
-          level: loggingEvent.level.level / 100,
-          level_name: level,
-          channel: logChannel,
-          datetime: new Date(loggingEvent.startTime).toISOString()
-        }
-      ];
-
-      this.send(logstashEvent);
+      const {time, ...props} = loggingEvent.getData();
+
+      this.send({
+        ...props,
+        message: format(loggingEvent.getMessage()),
+        context: loggingEvent.context.toJSON(),
+        level: loggingEvent.level.level / 100,
+        level_name: level,
+        channel: logChannel,
+        datetime: new Date(time || loggingEvent.startTime).toISOString()
+      });
     }
   }
 
   send(bulk: Record<string, any>) {
-    const {bufferMax = 0} = this.config.options;
+    const {bufferMax = 0, delayToFlush = 0} = this.config.options;
     this.#buffer.push(bulk);
 
+    if (delayToFlush) {
+      this.#timer && clearTimeout(this.#timer);
+      this.#timer = setTimeout(() => this.flush(), delayToFlush);
+    }
+
     if (bufferMax <= this.#buffer.length) {
       this.#buffer.push(bulk);
       return this.flush();
     }
   }
 
-  flush() {
+  async flush() {
     // send to server
     const buffer = this.#buffer;
     this.#buffer = [];
-
     if (buffer.length) {
-      const {url} = this.config.options;
-      const {application, logType} = this.config.options;
-
-      const header = JSON.stringify({
-        index: {
-          _index: typeof application === "function" ? application() : application,
-          _type: logType
-        }
-      });
+      return this;
+    }
 
-      const bulkData =
-        buffer
-          .flatMap((obj) => {
-            return [header, JSON.stringify(obj)];
-          }, [])
-          .join("\n") + "\n";
-
-      return this.client.post("", bulkData).catch((error) => {
-        if (error.response) {
-          console.error(
-            `Ts.ED Logger.logstash-http Appender error posting to ${url}: ${error.response.status} - ${JSON.stringify(error.response.data)}`
-          );
-          return;
+    const {url} = this.config.options;
+    const {application, logType} = this.config.options;
+    const _index = typeof application === "function" ? application() : application;
+
+    const action = JSON.stringify({
+      index: {
+        _index,
+        _type: logType
+      }
+    });
+
+    const bulkData = buffer.flatMap((item) => [action, item]);
+    try {
+      await this.client({
+        url: "",
+        method: "POST",
+        data: this.serializeBulk(bulkData),
+        headers: {
+          "Content-Type": "application/x-ndjson"
         }
-        console.error(`Ts.ED Logger.logstash-http Appender error: ${error.message}`);
       });
+    } catch (error) {
+      if (error.response) {
+        console.error(
+          `Ts.ED Logger.logstash-http Appender error posting to ${url}: ${error.response.status} - ${JSON.stringify(error.response.data)}`
+        );
+        return;
+      }
+      console.error(`Ts.ED Logger.logstash-http Appender error: ${error.message}`);
     }
   }
 
+  serializeBulk(array: Array<Record<string, any> | string>): string {
+    return array.reduce<string>((ndjson, obj) => {
+      const str = typeof obj === "string" ? obj : JSON.stringify(obj);
+
+      return ndjson + str + "\n";
+    }, "");
+  }
+
   shutdown() {
     return this.flush();
   }

packages/loggly/package.json

@@ -48,4 +48,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/logstash-http/package.json

@@ -49,4 +49,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/logstash-http/src/LogStashHttpAppender.ts

@@ -48,4 +48,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/logstash-udp/package.json

@@ -48,4 +48,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}

packages/rabbitmq/package.json

@@ -48,4 +48,4 @@
   "peerDependencies": {
     "@tsed/logger": "6.3.0"
   }
-}
+}