diff --git a/package-lock.json b/package-lock.json index 9cd19530..2ced1555 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4286,6 +4286,10 @@ "resolved": "packages/hooks", "link": true }, + "node_modules/@flatfile/http-logger": { + "resolved": "packages/http-logger", + "link": true + }, "node_modules/@flatfile/javascript": { "resolved": "packages/javascript", "link": true @@ -35992,7 +35996,7 @@ }, "packages/cli": { "name": "flatfile", - "version": "3.9.2", + "version": "3.10.0", "dependencies": { "@flatfile/cross-env-config": "^0.0.6", "@flatfile/listener": "^1.0.4", @@ -37540,6 +37544,29 @@ "tsup": "^6.1.3" } }, + "packages/http-logger": { + "name": "@flatfile/http-logger", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@flatfile/utils-debugger": "^0.0.6", + "strip-ansi": "^7.1.0" + } + }, + "packages/http-logger/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "packages/javascript": { "name": "@flatfile/javascript", "version": "1.5.6", @@ -39650,7 +39677,7 @@ }, "packages/listener": { "name": "@flatfile/listener", - "version": "1.1.1", + "version": "1.1.2", "license": "MIT", "dependencies": { "ansi-colors": "^4.1.3", diff --git a/packages/http-logger/index.d.ts b/packages/http-logger/index.d.ts new file mode 100644 index 00000000..a08038d7 --- /dev/null +++ b/packages/http-logger/index.d.ts @@ -0,0 +1,79 @@ +/** + * HTTP request log data interface + */ +export interface HttpLogData { + /** + * Whether the request resulted in an error + */ + error?: boolean + + /** + * HTTP method used (GET, POST, etc.) + */ + method: string + + /** + * Request URL + */ + url: string + + /** + * When the request started + */ + startTime: Date + + /** + * Response headers + */ + headers?: Record | Headers + + /** + * Response status code + */ + statusCode: number + + /** + * Size of request in bytes + */ + requestSize?: number + + /** + * Size of response in bytes + */ + responseSize?: number + + /** + * Whether this is a streaming response + */ + isStreaming?: boolean +} + +/** + * Global logger type definition for extending NodeJS.Global + */ +declare global { + var httpLogger: + | ((logData: { + url: string + method: string + duration: number + statusCode: number + requestSize: number + responseSize: number + isStreaming?: boolean + }) => void) + | undefined +} + +/** + * Logs HTTP request details to both Flatfile's internal debugger and any global HTTP logger. + * @param logData - HTTP request log data + */ +export function logHttpRequest(logData: HttpLogData): void + +/** + * Instruments HTTP requests to log them to the console. + * Automatically patches Node's HTTP/HTTPS modules and global fetch to track and log requests. + * This is useful for debugging HTTP interactions. + */ +export function instrumentRequests(): void diff --git a/packages/http-logger/index.js b/packages/http-logger/index.js new file mode 100644 index 00000000..2922f377 --- /dev/null +++ b/packages/http-logger/index.js @@ -0,0 +1,366 @@ +// @ts-nocheck + +import { Debugger } from "@flatfile/utils-debugger" +import stripAnsi from "strip-ansi" + +/** + * Unified HTTP request logger function that logs to both Debugger.logHttpRequest and global.httpLogger + * @param {Object} logData - HTTP request log data + * @param {boolean} [logData.error] - Whether the request resulted in an error + * @param {string} logData.method - HTTP method used + * @param {string} logData.url - Request URL + * @param {Date} logData.startTime - When the request started + * @param {Object} [logData.headers] - Request headers + * @param {number} logData.statusCode - Response status code + * @param {number} [logData.requestSize] - Size of request in bytes + * @param {number} [logData.responseSize] - Size of response in bytes + * @param {boolean} [logData.isStreaming] - Whether this is a streaming response + */ +export function logHttpRequest(logData) { + // Log to global.httpLogger if it exists + if (typeof global.httpLogger === "function") { + const endTime = new Date() + const duration = endTime.getTime() - logData.startTime.getTime() + + global.httpLogger({ + url: logData.url, + method: logData.method, + duration, + statusCode: logData.statusCode, + requestSize: logData.requestSize || 0, + responseSize: logData.responseSize || 0, + isStreaming: logData.isStreaming, + }) + } else { + // Log to Debugger + Debugger.logHttpRequest(logData) + } +} + +/** + * Gets size from headers if available + * @param {Object} headers - Headers object + * @returns {number|undefined} Size in bytes or undefined if not available + */ +function getSizeFromHeaders(headers) { + if (!headers) return undefined + + // Check for content-length in various formats (case insensitive) + const contentLength = + headers["content-length"] || + headers["Content-Length"] || + (typeof headers.get === "function" && headers.get("content-length")) + + if (contentLength) { + const size = parseInt(contentLength, 10) + return isNaN(size) ? undefined : size + } + + return undefined +} + +/** + * Checks if the response is likely to be a streaming response + * @param {Object} headers - Response headers + * @returns {boolean} True if it's likely a streaming response + */ +function isLikelyStreaming(headers) { + if (!headers) return false + + // Check for streaming indicators in headers + const transferEncoding = + headers["transfer-encoding"] || + headers["Transfer-Encoding"] || + (typeof headers.get === "function" && headers.get("transfer-encoding")) + + if (transferEncoding && transferEncoding.toLowerCase() === "chunked") { + return true + } + + // Check for content-type that indicates streaming + const contentType = + headers["content-type"] || + headers["Content-Type"] || + (typeof headers.get === "function" && headers.get("content-type")) + + if (contentType) { + const type = contentType.toLowerCase() + return type.includes("stream") || + type.includes("event-stream") || + type.includes("octet-stream") + } + + return false +} + +/** + * Calculates the size of a request body + * @param {any} data - Request body data + * @returns {number} Size in bytes + */ +function calculateRequestSize(data) { + if (!data) return 0 + + if (typeof data === "string") { + return Buffer.byteLength(data) + } else if (data instanceof Buffer) { + return data.length + } else if (data instanceof URLSearchParams) { + return Buffer.byteLength(data.toString()) + } else if (data instanceof FormData || data instanceof Blob) { + // We can't directly measure these, so we'll estimate + try { + return data.size || 0 + } catch (e) { + return 0 + } + } else if (typeof data === "object") { + try { + return Buffer.byteLength(JSON.stringify(data)) + } catch (e) { + return 0 + } + } + + return 0 +} + +/** + * Instruments HTTP requests to log them to the console. + * This is useful for debugging + */ +export function instrumentRequests() { + global.__instrumented = global.__instrumented === undefined ? false : global.__instrumented + + if (global.__instrumented) { + return + } else { + global.__instrumented = true + } + + if (!!process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.CI === "true") { + const olog = console.log + console.log = (farg, ...args) => olog(typeof farg === "string" ? stripAnsi(farg) : farg, ...args) + } + + function requestLogger(httpModule) { + const original = httpModule.request + if (!httpModule.__instrumented) { + httpModule.__instrumented = true + httpModule.request = function (options, callback) { + const host = options.host || options.hostname || options.href + const protocol = options.protocol || options.proto || options.href?.split(":").shift() || 'http' + // Remove trailing colon from protocol if it exists + const cleanProtocol = protocol.replace(/:$/, '') + + if ((host)?.includes("pndsn") || (host)?.endsWith("/ack")) { + return original(options, callback) + } + const startTime = new Date() + + // Try to get request size from Content-Length header first + let requestSize = getSizeFromHeaders(options.headers) + + const request = original.apply(this, [options, callback]) + + // If no Content-Length header, track request size manually without breaking streaming + if (requestSize === undefined) { + requestSize = 0 + const originalWrite = request.write + request.write = function (chunk, encoding, callback) { + if (chunk) { + const chunkSize = Buffer.isBuffer(chunk) ? chunk.length : Buffer.byteLength(chunk, encoding || 'utf8') + requestSize += chunkSize + } + return originalWrite.apply(this, arguments) + } + + // Also track the end method to ensure we catch all data + const originalEnd = request.end + request.end = function(chunk, encoding, callback) { + if (chunk) { + const chunkSize = Buffer.isBuffer(chunk) ? chunk.length : Buffer.byteLength(chunk, encoding || 'utf8') + requestSize += chunkSize + } + return originalEnd.apply(this, arguments) + } + } + + request.on("response", (response) => { + // Check if this is likely a streaming response + const isStreaming = isLikelyStreaming(response.headers) + + // Try to get response size from Content-Length header + let responseSize = getSizeFromHeaders(response.headers) + + // For non-streaming responses without Content-Length, track size + // but for streaming ones, we'll just mark it as streaming + if (responseSize === undefined && !isStreaming) { + responseSize = 0 + response.on("data", (chunk) => { + responseSize += chunk.length + }) + } + + // Log immediately when headers are received for streaming responses + if (isStreaming) { + logHttpRequest({ + error: response.statusCode >= 400, + method: options.method, + url: options.href || cleanProtocol + "://" + host + options.path, + startTime, + headers: response.headers, + statusCode: response.statusCode, + requestSize, + responseSize: responseSize || 0, + isStreaming: true, + }) + } else { + // For non-streaming, log when response is complete + response.on("end", () => { + logHttpRequest({ + error: response.statusCode >= 400, + method: options.method, + url: options.href || cleanProtocol + "://" + host + options.path, + startTime, + headers: response.headers, + statusCode: response.statusCode, + requestSize, + responseSize, + isStreaming: false, + }) + }) + } + }) + return request + } + } + } + + // eslint-disable-next-line + requestLogger(require("http")) + // eslint-disable-next-line + requestLogger(require("https")) + + // Instrumenting fetch + const originalFetch = globalThis.fetch + globalThis.fetch = async (input, init) => { + const startTime = new Date() + let method = "GET" + let url = "" + let headers = {} + + if (typeof input === "string") { + url = input + method = init?.method || "GET" + headers = init?.headers || {} + } else if (typeof input === "object") { + url = input.url || "" + method = input.method || "GET" + headers = input.headers || {} + } + + // Try to get request size from Content-Length header first + let requestSize = getSizeFromHeaders(headers) + + // If no Content-Length header, try to calculate from body + if (requestSize === undefined) { + if (typeof input === "string" && init?.body) { + requestSize = calculateRequestSize(init.body) + } else if (typeof input === "object" && input.body) { + requestSize = calculateRequestSize(input.body) + } else { + requestSize = 0 + } + } + + try { + const response = await originalFetch(input, init) + + // Check if this is likely a streaming response + const isStreaming = isLikelyStreaming(response.headers) || + (response.bodyUsed === false && response.body && + typeof response.body.getReader === 'function'); + + // Try to get response size from Content-Length header + let responseSize = getSizeFromHeaders(response.headers) + + // For non-streaming responses without Content-Length, try to safely measure size + // but ONLY if the response is not streaming and hasn't been used yet + if (responseSize === undefined && !isStreaming && + response.bodyUsed === false && response.clone && + typeof response.clone === "function") { + try { + // Only attempt to measure size for small responses (< 10MB) + const contentType = response.headers.get('content-type') || ''; + const isBinary = contentType.includes('image/') || + contentType.includes('audio/') || + contentType.includes('video/') || + contentType.includes('application/octet-stream'); + + // Skip size measurement for binary content or when we suspect large files + if (!isBinary) { + const clonedResponse = response.clone(); + + // Set a size limit to avoid memory issues (10MB) + const MAX_SIZE = 10 * 1024 * 1024; + const reader = clonedResponse.body.getReader(); + let bytesRead = 0; + let done = false; + + while (!done && bytesRead < MAX_SIZE) { + const { value, done: doneReading } = await reader.read(); + done = doneReading; + if (value) { + bytesRead += value.length; + } + + // If we hit the limit, stop measuring + if (bytesRead >= MAX_SIZE) { + break; + } + } + + responseSize = bytesRead; + + // If we hit the limit, mark this as an approximate size + if (bytesRead >= MAX_SIZE) { + responseSize = MAX_SIZE; // Indicate we hit the limit + } + } + } catch (e) { + // If reading the response fails, we'll just log zero size + responseSize = 0; + } + } + + const logDetails = { + error: !response.ok, + method, + url, + startTime, + headers: Object.fromEntries(response.headers.entries()), + statusCode: response.status, + requestSize, + responseSize: responseSize || 0, + isStreaming: isStreaming, + } + + logHttpRequest(logDetails) + + return response + } catch (error) { + logHttpRequest({ + error: true, + method, + url, + startTime, + headers: {}, + statusCode: 0, + requestSize, + responseSize: 0, + }) + throw error + } + } +} diff --git a/packages/http-logger/init.d.ts b/packages/http-logger/init.d.ts new file mode 100644 index 00000000..f6f0b654 --- /dev/null +++ b/packages/http-logger/init.d.ts @@ -0,0 +1,4 @@ +// Re-export everything from the main module +export * from './index' + +// Note: instrumentRequests() is automatically called when this module is imported diff --git a/packages/http-logger/init.js b/packages/http-logger/init.js new file mode 100644 index 00000000..48959b02 --- /dev/null +++ b/packages/http-logger/init.js @@ -0,0 +1,9 @@ +// @ts-nocheck + +import { instrumentRequests } from './index.js' + +// Automatically instrument HTTP requests when this module is imported +instrumentRequests() + +// Re-export everything from the main module +export * from './index.js' diff --git a/packages/http-logger/package.json b/packages/http-logger/package.json new file mode 100644 index 00000000..1e781ccc --- /dev/null +++ b/packages/http-logger/package.json @@ -0,0 +1,26 @@ +{ + "name": "@flatfile/http-logger", + "version": "1.0.4", + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "./init": { + "types": "./init.d.ts", + "default": "./init.js" + } + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + "@flatfile/utils-debugger": "^0.0.6", + "strip-ansi": "^7.1.0" + }, + "author": "", + "license": "ISC", + "description": "A lightweight, flexible HTTP request logger for Node.js applications" +} diff --git a/packages/http-logger/readme.md b/packages/http-logger/readme.md new file mode 100644 index 00000000..0426865e --- /dev/null +++ b/packages/http-logger/readme.md @@ -0,0 +1,118 @@ +# @flatfile/http-logger + +A lightweight, flexible HTTP request logger for Node.js applications that provides detailed logging for HTTP requests. + +## Features + +- 🔍 Automatically logs all HTTP/HTTPS requests and fetch API calls +- 🔄 Tracks request and response sizes, durations, and status codes +- 📊 Special handling for streaming responses +- ⚡ Minimal performance impact +- 🧩 Works with both Node.js HTTP/HTTPS modules and global fetch +- 🔌 Integrates with Flatfile's debugging tools +- 📝 Full TypeScript support with type definitions + +## Installation + +```bash +# npm +npm install @flatfile/http-logger + +# yarn +yarn add @flatfile/http-logger + +# pnpm +pnpm add @flatfile/http-logger + +# bun +bun add @flatfile/http-logger +``` + +## Usage + +### Auto-Initialization + +The simplest way to use this library is with the auto-initializing import that automatically instruments all HTTP requests: + +```javascript +// Import the /init version to automatically instrument requests +import '@flatfile/http-logger/init'; + +// That's it! All HTTP requests will now be automatically logged +``` + +### Manual Initialization + +Alternatively, you can explicitly initialize the logger: + +```javascript +import { instrumentRequests } from '@flatfile/http-logger'; + +// Call this early in your application startup +instrumentRequests(); + +// All HTTP requests will now be automatically logged +``` + +### Manual Logging + +You can also manually log HTTP requests: + +```javascript +import { logHttpRequest } from '@flatfile/http-logger'; + +// Log a request manually +logHttpRequest({ + method: 'GET', + url: 'https://api.example.com/data', + startTime: new Date(), // when the request started + statusCode: 200, + headers: { 'content-type': 'application/json' }, + requestSize: 0, + responseSize: 1024, + isStreaming: false +}); +``` + +## API + +### `instrumentRequests()` + +Patches the native HTTP modules and fetch API to automatically log all HTTP requests. This should be called early in your application. + +```javascript +import { instrumentRequests } from '@flatfile/http-logger'; +instrumentRequests(); +``` + +### `logHttpRequest(logData)` + +Logs HTTP request details to both Flatfile's internal debugger and any global HTTP logger. + +**Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `logData.error` | boolean | Whether the request resulted in an error | +| `logData.method` | string | HTTP method used (GET, POST, etc.) | +| `logData.url` | string | Request URL | +| `logData.startTime` | Date | When the request started | +| `logData.headers` | Object | Response headers | +| `logData.statusCode` | number | Response status code | +| `logData.requestSize` | number | Size of request in bytes | +| `logData.responseSize` | number | Size of response in bytes | +| `logData.isStreaming` | boolean | Whether this is a streaming response | + +## How it Works + +The library patches Node's native HTTP/HTTPS modules and the global fetch API to intercept requests and responses. It calculates timing and size information, and handles streaming responses appropriately. + +For streaming responses, it logs information as soon as headers are received, while for regular responses it waits until the full response is received. + +## Environment Handling + +In AWS Lambda or CI environments, ANSI color codes are automatically stripped from console output to improve log readability. + +## License + +MIT