feat: support option for logging minimal data

lifeomic · Nov 29, 2023 · 3134bae · 3134bae
1 parent bc74e1c
commit 3134bae
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 11 deletions.
diff --git a/src/dynamo-streams.ts b/src/dynamo-streams.ts
@@ -216,7 +216,9 @@ export class DynamoStreamHandler<Entity, Context> {
       };
 
       context.logger.info(
-        { event: this.obfuscateEvent(event) },
+        this.config.useMinimalLogging
+          ? { eventIds: event.Records.map((r) => r.eventID) }
+          : { event: this.obfuscateEvent(event) },
         'Processing DynamoDB stream event',
       );
 
@@ -249,9 +251,11 @@ export class DynamoStreamHandler<Entity, Context> {
           concurrency: this.config.concurrency ?? 5,
         },
         async (record) => {
-          const recordLogger = this.config.logger.child({
-            record: this.obfuscateRecord(record),
-          });
+          const recordLogger = this.config.logger.child(
+            this.config.useMinimalLogging
+              ? { recordEventId: record.eventID }
+              : { record: this.obfuscateRecord(record) },
+          );
           if (!record.dynamodb) {
             recordLogger.error(
               'The dynamodb property was not present on event',

diff --git a/src/kinesis.ts b/src/kinesis.ts
@@ -78,6 +78,12 @@ export class KinesisEventHandler<Event, Context> {
       Object.assign(context, await this.config.createRunContext(context));
 
       // 2. Process all the records.
+      context.logger.info(
+        this.config.useMinimalLogging
+          ? { eventIds: event.Records.map((r) => r.eventID) }
+          : { event },
+        'Processing Kinesis event',
+      );
       const processingResult = await processWithOrdering(
         {
           items: event.Records,

diff --git a/src/sqs.ts b/src/sqs.ts
@@ -96,7 +96,12 @@ export class SQSMessageHandler<Message, Context> {
       Object.assign(context, await this.config.createRunContext(context));
 
       // 2. Process all the records.
-      context.logger.info({ event }, 'Processing SQS topic message');
+      context.logger.info(
+        this.config.useMinimalLogging
+          ? { messageIds: event.Records.map((r) => r.messageId) }
+          : { event },
+        'Processing SQS message',
+      );
 
       const processingResult = await processWithOrdering(
         {
@@ -140,12 +145,20 @@ export class SQSMessageHandler<Message, Context> {
         .map(([groupId, record]) => {
           const [failedRecord, ...subsequentUnprocessedRecords] = record.items;
           context.logger.error(
-            {
-              groupId,
-              err: record.error,
-              failedRecord,
-              subsequentUnprocessedRecords,
-            },
+            this.config.useMinimalLogging
+              ? {
+                  groupId,
+                  err: record.error,
+                  failedRecordMessageId: failedRecord.messageId,
+                  subsequentUnprocessedRecordsMessageIds:
+                    subsequentUnprocessedRecords.map((r) => r.messageId),
+                }
+              : {
+                  groupId,
+                  err: record.error,
+                  failedRecord,
+                  subsequentUnprocessedRecords,
+                },
             'Failed to fully process message group',
           );
 

diff --git a/src/utils.ts b/src/utils.ts
@@ -19,6 +19,13 @@ export type BaseHandlerConfig<Context> = {
    */
   createRunContext: (base: BaseContext) => Context | Promise<Context>;
 
+  /**
+   * Whether to minimize internal logging of events. When set to `true`,
+   * the handler will never log the full content of events, and will
+   * instead only log unique identifiers for the events.
+   */
+  useMinimalLogging?: boolean;
+
   /**
    * The maximum concurrency for processing events.
    *