feat: support option for logging minimal data

lifeomic · Nov 29, 2023 · 165adb7 · 165adb7
1 parent f1d23a9
commit 165adb7
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 11 deletions.
diff --git a/src/dynamo-streams.ts b/src/dynamo-streams.ts
@@ -34,6 +34,9 @@ export type DynamoStreamHandlerConfig<Entity, Context> = {
    * Create a "context" for the lambda execution. (e.g. "data sources")
    */
   createRunContext: (base: BaseContext) => Context | Promise<Context>;
+
+  useMinimalLogging?: boolean;
+
   /**
    * The maximum concurrency for processing records.
    *
@@ -227,7 +230,9 @@ export class DynamoStreamHandler<Entity, Context> {
       };
 
       context.logger.info(
-        { event: this.obfuscateEvent(event) },
+        this.config.useMinimalLogging
+          ? { eventIds: event.Records.map((r) => r.eventID) }
+          : { event: this.obfuscateEvent(event) },
         'Processing DynamoDB stream event',
       );
 
@@ -260,9 +265,11 @@ export class DynamoStreamHandler<Entity, Context> {
           concurrency: this.config.concurrency ?? 5,
         },
         async (record) => {
-          const recordLogger = this.config.logger.child({
-            record: this.obfuscateRecord(record),
-          });
+          const recordLogger = this.config.logger.child(
+            this.config.useMinimalLogging
+              ? { recordEventId: record.eventID }
+              : { record: this.obfuscateRecord(record) },
+          );
           if (!record.dynamodb) {
             recordLogger.error(
               'The dynamodb property was not present on event',

diff --git a/src/kinesis.ts b/src/kinesis.ts
@@ -20,6 +20,8 @@ export type KinesisEventHandlerConfig<Event, Context> = {
    * Create a "context" for the lambda execution. (e.g. "data sources")
    */
   createRunContext: (base: BaseContext) => Context | Promise<Context>;
+
+  useMinimalLogging?: boolean;
   /**
    * The maximum concurrency for processing events.
    *
@@ -90,6 +92,12 @@ export class KinesisEventHandler<Event, Context> {
       Object.assign(context, await this.config.createRunContext(context));
 
       // 2. Process all the records.
+      context.logger.info(
+        this.config.useMinimalLogging
+          ? { eventIds: event.Records.map((r) => r.eventID) }
+          : { event },
+        'Processing Kinesis event',
+      );
       const processingResult = await processWithOrdering(
         {
           items: event.Records,

diff --git a/src/sqs.ts b/src/sqs.ts
@@ -20,6 +20,9 @@ export type SQSMessageHandlerConfig<Message, Context> = {
    * Create a "context" for the lambda execution. (e.g. "data sources")
    */
   createRunContext: (base: BaseContext) => Context | Promise<Context>;
+
+  useMinimalLogging?: boolean;
+
   /**
    * The maximum concurrency for processing messages.
    *
@@ -107,7 +110,12 @@ export class SQSMessageHandler<Message, Context> {
       Object.assign(context, await this.config.createRunContext(context));
 
       // 2. Process all the records.
-      context.logger.info({ event }, 'Processing SQS topic message');
+      context.logger.info(
+        this.config.useMinimalLogging
+          ? { messageIds: event.Records.map((r) => r.messageId) }
+          : { event },
+        'Processing SQS message',
+      );
 
       const processingResult = await processWithOrdering(
         {
@@ -151,12 +159,20 @@ export class SQSMessageHandler<Message, Context> {
         .map(([groupId, record]) => {
           const [failedRecord, ...subsequentUnprocessedRecords] = record.items;
           context.logger.error(
-            {
-              groupId,
-              err: record.error,
-              failedRecord,
-              subsequentUnprocessedRecords,
-            },
+            this.config.useMinimalLogging
+              ? {
+                  groupId,
+                  err: record.error,
+                  failedRecordMessageId: failedRecord.messageId,
+                  subsequentUnprocessedRecordsMessageIds:
+                    subsequentUnprocessedRecords.map((r) => r.messageId),
+                }
+              : {
+                  groupId,
+                  err: record.error,
+                  failedRecord,
+                  subsequentUnprocessedRecords,
+                },
             'Failed to fully process message group',
           );