node: overhaul of database and storage [INT-355]

Author: perf2711

packages/node/src/BacktraceClient.ts

@@ -7,7 +7,7 @@ import {
     SessionFiles,
     VariableDebugIdMapProvider,
 } from '@backtrace/sdk-core';
-import path from 'path';
+import nodeFs from 'fs';
 import { BacktraceConfiguration, BacktraceSetupConfiguration } from './BacktraceConfiguration.js';
 import { BacktraceNodeRequestHandler } from './BacktraceNodeRequestHandler.js';
 import { AGENT } from './agentDefinition.js';
@@ -17,39 +17,85 @@ import { FileBreadcrumbsStorage } from './breadcrumbs/FileBreadcrumbsStorage.js'
 import { BacktraceClientBuilder } from './builder/BacktraceClientBuilder.js';
 import { BacktraceNodeClientSetup } from './builder/BacktraceClientSetup.js';
 import { NodeOptionReader } from './common/NodeOptionReader.js';
+import { toArray } from './common/asyncGenerator.js';
 import { NodeDiagnosticReportConverter } from './converter/NodeDiagnosticReportConverter.js';
-import { FsNodeFileSystem } from './storage/FsNodeFileSystem.js';
-import { NodeFileSystem } from './storage/interfaces/NodeFileSystem.js';
+import {
+    AttachmentBacktraceDatabaseRecordSender,
+    AttachmentBacktraceDatabaseRecordSerializer,
+} from './database/AttachmentBacktraceDatabaseRecord.js';
+import {
+    ReportBacktraceDatabaseRecordWithAttachmentsFactory,
+    ReportBacktraceDatabaseRecordWithAttachmentsSender,
+    ReportBacktraceDatabaseRecordWithAttachmentsSerializer,
+} from './database/ReportBacktraceDatabaseRecordWithAttachments.js';
+import { assertDatabasePath } from './database/utils.js';
+import { BacktraceStorageModule } from './storage/BacktraceStorage.js';
+import { BacktraceStorageModuleFactory } from './storage/BacktraceStorageModuleFactory.js';
+import { NodeFsBacktraceStorageModuleFactory } from './storage/NodeFsBacktraceStorage.js';
 
 export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration> {
     private _listeners: Record<string, NodeJS.UnhandledRejectionListener | NodeJS.UncaughtExceptionListener> = {};
 
-    protected get nodeFileSystem() {
-        return this.fileSystem as NodeFileSystem | undefined;
+    protected readonly storageFactory: BacktraceStorageModuleFactory;
+    protected readonly fs: typeof nodeFs;
+
+    protected get databaseNodeFsStorage() {
+        return this.databaseStorage as BacktraceStorageModule | undefined;
     }
 
     constructor(clientSetup: BacktraceNodeClientSetup) {
-        const fileSystem = clientSetup.fileSystem ?? new FsNodeFileSystem();
+        const storageFactory = clientSetup.storageFactory ?? new NodeFsBacktraceStorageModuleFactory();
+        const fs = clientSetup.fs ?? nodeFs;
+        const storage =
+            clientSetup.database?.storage ??
+            (clientSetup.options.database?.enable
+                ? storageFactory.create({
+                      path: assertDatabasePath(clientSetup.options.database.path),
+                      createDirectory: clientSetup.options.database.createDatabaseDirectory,
+                      fs,
+                  })
+                : undefined);
+
         super({
             sdkOptions: AGENT,
             requestHandler: new BacktraceNodeRequestHandler(clientSetup.options),
             debugIdMapProvider: new VariableDebugIdMapProvider(global as DebugIdContainer),
+            database:
+                clientSetup.options.database?.enable && storage
+                    ? {
+                          storage,
+                          reportRecordFactory: ReportBacktraceDatabaseRecordWithAttachmentsFactory.default(),
+                          ...clientSetup.database,
+                          recordSenders: (submission) => ({
+                              report: new ReportBacktraceDatabaseRecordWithAttachmentsSender(submission),
+                              attachment: new AttachmentBacktraceDatabaseRecordSender(submission),
+                              ...clientSetup.database?.recordSenders?.(submission),
+                          }),
+                          recordSerializers: {
+                              report: new ReportBacktraceDatabaseRecordWithAttachmentsSerializer(storage),
+                              attachment: new AttachmentBacktraceDatabaseRecordSerializer(fs),
+                              ...clientSetup.database?.recordSerializers,
+                          },
+                      }
+                    : undefined,
             ...clientSetup,
-            fileSystem,
             options: {
                 ...clientSetup.options,
                 attachments: clientSetup.options.attachments?.map(transformAttachment),
             },
         });
 
+        this.storageFactory = storageFactory;
+        this.fs = fs;
+
         const breadcrumbsManager = this.modules.get(BreadcrumbsManager);
-        if (breadcrumbsManager && this.sessionFiles) {
-            breadcrumbsManager.setStorage(FileBreadcrumbsStorage.factory(this.sessionFiles, fileSystem));
+        if (breadcrumbsManager && this.sessionFiles && storage) {
+            breadcrumbsManager.setStorage(FileBreadcrumbsStorage.factory(this.sessionFiles, storage));
         }
 
-        if (this.sessionFiles && clientSetup.options.database?.captureNativeCrashes) {
-            this.addModule(FileAttributeManager, FileAttributeManager.create(fileSystem));
-            this.addModule(FileAttachmentsManager, FileAttachmentsManager.create(fileSystem));
+        if (this.sessionFiles && storage && clientSetup.options.database?.captureNativeCrashes) {
+            this.addModule(FileAttributeManager, FileAttributeManager.create(storage));
+            this.addModule(FileAttachmentsManager, FileAttachmentsManager.create(storage));
         }
     }
 
@@ -58,6 +104,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
 
         try {
             super.initialize();
+
             this.captureUnhandledErrors(
                 this.options.captureUnhandledErrors,
                 this.options.captureUnhandledPromiseRejections,
@@ -242,18 +289,19 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
     }
 
     private async loadNodeCrashes() {
-        if (!this.database || !this.nodeFileSystem || !this.options.database?.captureNativeCrashes) {
+        if (!this.database || !this.options.database?.captureNativeCrashes) {
             return;
         }
 
         const reportName = process.report?.filename;
-        const databasePath = process.report?.directory
-            ? process.report.directory
-            : (this.options.database?.path ?? process.cwd());
+        const storage = this.storageFactory.create({
+            path: process.report?.directory ? process.report.directory : (this.options.database?.path ?? process.cwd()),
+            fs: this.fs,
+        });
 
         let databaseFiles: string[];
         try {
-            databaseFiles = await this.nodeFileSystem.readDir(databasePath);
+            databaseFiles = await toArray(storage.keys());
         } catch {
             return;
         }
@@ -271,11 +319,14 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
 
         const reports: [path: string, report: BacktraceReport, sessionFiles?: SessionFiles][] = [];
         for (const recordName of recordNames) {
-            const recordPath = path.join(databasePath, recordName);
             try {
-                const recordJson = await this.nodeFileSystem.readFile(recordPath);
+                const recordJson = await storage.get(recordName);
+                if (!recordJson) {
+                    continue;
+                }
+
                 const report = converter.convert(JSON.parse(recordJson));
-                reports.push([recordPath, report]);
+                reports.push([recordName, report]);
             } catch {
                 // Do nothing, skip the report
             }
@@ -292,17 +343,15 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
             currentSession = currentSession?.getPreviousSession();
         }
 
-        for (const [recordPath, report, session] of reports) {
+        for (const [recordName, report, session] of reports) {
             try {
                 if (session) {
-                    report.attachments.push(
-                        ...FileBreadcrumbsStorage.getSessionAttachments(session, this.nodeFileSystem),
-                    );
+                    report.attachments.push(...FileBreadcrumbsStorage.getSessionAttachments(session, storage));
 
-                    const fileAttributes = FileAttributeManager.createFromSession(session, this.nodeFileSystem);
+                    const fileAttributes = FileAttributeManager.createFromSession(session, storage);
                     Object.assign(report.attributes, await fileAttributes.get());
 
-                    const fileAttachments = FileAttachmentsManager.createFromSession(session, this.nodeFileSystem);
+                    const fileAttachments = FileAttachmentsManager.createFromSession(session, storage);
                     report.attachments.push(...(await fileAttachments.get()));
 
                     report.attributes['application.session'] = session.sessionId;
@@ -318,7 +367,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
                 // Do nothing, skip the report
             } finally {
                 try {
-                    await this.nodeFileSystem.unlink(recordPath);
+                    await storage.remove(recordName);
                 } catch {
                     // Do nothing
                 }

packages/node/src/BacktraceConfiguration.ts

@@ -1,10 +1,37 @@
-import { BacktraceAttachment, BacktraceConfiguration as CoreConfiguration } from '@backtrace/sdk-core';
+import {
+    BacktraceAttachment,
+    BacktraceConfiguration as CoreConfiguration,
+    DisabledBacktraceDatabaseConfiguration as CoreDisabledBacktraceDatabaseConfiguration,
+    EnabledBacktraceDatabaseConfiguration as CoreEnabledBacktraceDatabaseConfiguration,
+} from '@backtrace/sdk-core';
 import { Readable } from 'stream';
 
-export interface BacktraceSetupConfiguration extends Omit<CoreConfiguration, 'attachments'> {
+export interface EnabledBacktraceDatabaseConfiguration extends CoreEnabledBacktraceDatabaseConfiguration {
+    /**
+     * Path where the SDK can store data.
+     */
+    path: string;
+    /**
+     * Determine if the directory should be auto created by the SDK.
+     * @default true
+     */
+    createDatabaseDirectory?: boolean;
+}
+
+export interface DisabledBacktraceDatabaseConfiguration
+    extends CoreDisabledBacktraceDatabaseConfiguration,
+        Omit<Partial<EnabledBacktraceDatabaseConfiguration>, 'enable'> {}
+
+export type BacktraceDatabaseConfiguration =
+    | EnabledBacktraceDatabaseConfiguration
+    | DisabledBacktraceDatabaseConfiguration;
+
+export interface BacktraceSetupConfiguration extends Omit<CoreConfiguration, 'attachments' | 'database'> {
     attachments?: Array<BacktraceAttachment<Buffer | Readable | string | Uint8Array> | string>;
+    database?: BacktraceDatabaseConfiguration;
 }
 
-export interface BacktraceConfiguration extends Omit<CoreConfiguration, 'attachments'> {
+export interface BacktraceConfiguration extends Omit<CoreConfiguration, 'attachments' | 'database'> {
     attachments?: BacktraceAttachment<Buffer | Readable | string | Uint8Array>[];
+    database?: BacktraceDatabaseConfiguration;
 }

packages/node/src/attachment/BacktraceFileAttachment.ts

@@ -1,24 +1,31 @@
-import { BacktraceFileAttachment as CoreBacktraceFileAttachment } from '@backtrace/sdk-core';
+import { BacktraceSyncStorage, BacktraceFileAttachment as CoreBacktraceFileAttachment } from '@backtrace/sdk-core';
 import fs from 'fs';
 import path from 'path';
 import { Readable } from 'stream';
-import { NodeFileSystem } from '../storage/interfaces/NodeFileSystem.js';
+import { BacktraceStreamStorage } from '../storage/BacktraceStorage.js';
 
 export class BacktraceFileAttachment implements CoreBacktraceFileAttachment<Readable> {
     public readonly name: string;
 
     constructor(
         public readonly filePath: string,
         name?: string,
-        private readonly _fileSystem?: NodeFileSystem,
+        private readonly _fs: typeof fs | (BacktraceSyncStorage & BacktraceStreamStorage) = fs,
     ) {
         this.name = name ?? path.basename(this.filePath);
     }
 
     public get(): Readable | undefined {
-        if (!(this._fileSystem ?? fs).existsSync(this.filePath)) {
-            return undefined;
+        if ('hasSync' in this._fs) {
+            if (!this._fs.hasSync(this.filePath)) {
+                return undefined;
+            }
+        } else {
+            if (!this._fs.existsSync(this.filePath)) {
+                return undefined;
+            }
         }
-        return (this._fileSystem ?? fs).createReadStream(this.filePath);
+
+        return this._fs.createReadStream(this.filePath);
     }
 }

packages/node/src/attachment/FileAttachmentsManager.ts

@@ -2,7 +2,7 @@ import {
     AttachmentManager,
     BacktraceModule,
     BacktraceModuleBindData,
-    FileSystem,
+    BacktraceStorage,
     SessionFiles,
 } from '@backtrace/sdk-core';
 import { BacktraceFileAttachment } from './BacktraceFileAttachment.js';
@@ -15,15 +15,15 @@ export class FileAttachmentsManager implements BacktraceModule {
     private _attachmentsManager?: AttachmentManager;
 
     constructor(
-        private readonly _fileSystem: FileSystem,
+        private readonly _storage: BacktraceStorage,
         private _fileName?: string,
     ) {}
 
-    public static create(fileSystem: FileSystem) {
-        return new FileAttachmentsManager(fileSystem);
+    public static create(storage: BacktraceStorage) {
+        return new FileAttachmentsManager(storage);
     }
 
-    public static createFromSession(sessionFiles: SessionFiles, fileSystem: FileSystem) {
+    public static createFromSession(sessionFiles: SessionFiles, fileSystem: BacktraceStorage) {
         const fileName = sessionFiles.getFileName(ATTACHMENT_FILE_NAME);
         return new FileAttachmentsManager(fileSystem, fileName);
     }
@@ -56,7 +56,10 @@ export class FileAttachmentsManager implements BacktraceModule {
         }
 
         try {
-            const content = await this._fileSystem.readFile(this._fileName);
+            const content = await this._storage.get(this._fileName);
+            if (!content) {
+                return [];
+            }
             const attachments = JSON.parse(content) as SavedAttachment[];
             return attachments.map(([path, name]) => new BacktraceFileAttachment(path, name));
         } catch {
@@ -74,6 +77,6 @@ export class FileAttachmentsManager implements BacktraceModule {
             .filter((f): f is BacktraceFileAttachment => f instanceof BacktraceFileAttachment)
             .map<SavedAttachment>((f) => [f.filePath, f.name]);
 
-        await this._fileSystem.writeFile(this._fileName, JSON.stringify(fileAttachments));
+        await this._storage.set(this._fileName, JSON.stringify(fileAttachments));
     }
 }

packages/node/src/attachment/isFileAttachment.ts

@@ -0,0 +1,9 @@
+import { BacktraceAttachment } from '@backtrace/sdk-core';
+import { BacktraceFileAttachment } from './BacktraceFileAttachment.js';
+
+export function isFileAttachment(attachment: BacktraceAttachment): attachment is BacktraceFileAttachment {
+    return (
+        attachment instanceof BacktraceFileAttachment ||
+        ('filePath' in attachment && typeof attachment.filePath === 'string')
+    );
+}

packages/node/src/breadcrumbs/FileBreadcrumbsStorage.ts

@@ -1,6 +1,8 @@
 import {
     BacktraceAttachment,
     BacktraceAttachmentProvider,
+    BacktraceStorage,
+    BacktraceSyncStorage,
     Breadcrumb,
     BreadcrumbLogLevel,
     BreadcrumbsStorage,
@@ -15,7 +17,7 @@ import {
 import path from 'path';
 import { Readable, Writable } from 'stream';
 import { BacktraceFileAttachment } from '../attachment/index.js';
-import { NodeFileSystem } from '../storage/interfaces/NodeFileSystem.js';
+import { BacktraceStreamStorage } from '../storage/BacktraceStorage.js';
 import { chunkifier, ChunkSplitterFactory } from '../streams/chunkifier.js';
 import { combinedChunkSplitter } from '../streams/combinedChunkSplitter.js';
 import { FileChunkSink } from '../streams/fileChunkSink.js';
@@ -36,7 +38,7 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage {
 
     constructor(
         session: SessionFiles,
-        private readonly _fileSystem: NodeFileSystem,
+        private readonly _storage: BacktraceStorage & BacktraceSyncStorage & BacktraceStreamStorage,
         private readonly _limits: BreadcrumbsStorageLimits,
     ) {
         const splitters: ChunkSplitterFactory[] = [];
@@ -52,7 +54,7 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage {
 
         this._sink = new FileChunkSink({
             maxFiles: 2,
-            fs: this._fileSystem,
+            storage: this._storage,
             file: (n) => session.getFileName(FileBreadcrumbsStorage.getFileName(n)),
         });
 
@@ -71,22 +73,28 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage {
         });
     }
 
-    public static getSessionAttachments(session: SessionFiles, fileSystem?: NodeFileSystem) {
+    public static getSessionAttachments(
+        session: SessionFiles,
+        storage?: BacktraceStorage & BacktraceSyncStorage & BacktraceStreamStorage,
+    ) {
         const files = session
             .getSessionFiles()
             .filter((f) => path.basename(f).startsWith(FILE_PREFIX))
             .slice(0, 2);
 
-        return files.map((file) => new BacktraceFileAttachment(file, path.basename(file), fileSystem));
+        return files.map((file) => new BacktraceFileAttachment(file, path.basename(file), storage));
     }
 
-    public static factory(session: SessionFiles, fileSystem: NodeFileSystem): BreadcrumbsStorageFactory {
-        return ({ limits }) => new FileBreadcrumbsStorage(session, fileSystem, limits);
+    public static factory(
+        session: SessionFiles,
+        storage: BacktraceStorage & BacktraceSyncStorage & BacktraceStreamStorage,
+    ): BreadcrumbsStorageFactory {
+        return ({ limits }) => new FileBreadcrumbsStorage(session, storage, limits);
     }
 
     public getAttachments(): BacktraceAttachment<Readable>[] {
         const files = [...this._sink.files].map((f) => f.path.toString('utf-8'));
-        return files.map((f) => new BacktraceFileAttachment(f, path.basename(f), this._fileSystem));
+        return files.map((f) => new BacktraceFileAttachment(f, path.basename(f), this._storage));
     }
 
     public getAttachmentProviders(): BacktraceAttachmentProvider[] {