From 33e60232bf115f1689f3fcc4e43b84898ff49df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Fri, 10 Oct 2025 08:15:27 +0200 Subject: [PATCH 1/3] feat(batch-snapshot): add warm-up feature to prevent cold start issues - Introduced ENABLE_WARM_UP environment variable to enable or disable API route warm-up. - Implemented warm-up logic in BatchSnapshotOrchestrator to make an OPTIONS request before processing batches. - Enhanced error handling for common cold start issues with specific messages for HTTP 405, 503, and 502 errors. - Updated documentation to reflect the new warm-up configuration option. --- .github/workflows/daily-balance-snapshots.yml | 3 +- scripts/batch-snapshot-orchestrator.ts | 73 ++++++++++++++++++- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml index b91fc47f..e55c4414 100644 --- a/.github/workflows/daily-balance-snapshots.yml +++ b/.github/workflows/daily-balance-snapshots.yml @@ -25,8 +25,6 @@ jobs: node-version: '18' cache: 'npm' - - name: Install dependencies - run: npm ci - name: Install script dependencies run: | @@ -44,6 +42,7 @@ jobs: DELAY_BETWEEN_BATCHES: 10 MAX_RETRIES: 3 REQUEST_TIMEOUT: 45 + ENABLE_WARM_UP: true - name: Notify on failure if: failure() diff --git a/scripts/batch-snapshot-orchestrator.ts b/scripts/batch-snapshot-orchestrator.ts index 5a5c83c9..12043535 100644 --- a/scripts/batch-snapshot-orchestrator.ts +++ b/scripts/batch-snapshot-orchestrator.ts @@ -18,6 +18,7 @@ * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 10) * - MAX_RETRIES: Maximum retries for failed batches (default: 3) * - REQUEST_TIMEOUT: Request timeout in seconds (default: 45) + * - ENABLE_WARM_UP: Enable API route warm-up to prevent cold start issues (default: true) */ interface BatchProgress { @@ -111,6 +112,7 @@ interface BatchConfig { delayBetweenBatches: number; maxRetries: number; requestTimeout: number; // in seconds + enableWarmUp: boolean; // whether to warm up API route before processing } interface ApiResponse { @@ -167,6 +169,9 @@ class BatchSnapshotOrchestrator { const delayBetweenBatches = this.parseAndValidateNumber(process.env.DELAY_BETWEEN_BATCHES || '10', 'DELAY_BETWEEN_BATCHES', 1, 300); const maxRetries = this.parseAndValidateNumber(process.env.MAX_RETRIES || '3', 'MAX_RETRIES', 1, 10); const requestTimeout = this.parseAndValidateNumber(process.env.REQUEST_TIMEOUT || '45', 'REQUEST_TIMEOUT', 10, 300); + + // Parse boolean environment variable for warm-up feature + const enableWarmUp = process.env.ENABLE_WARM_UP !== 'false'; // Default to true unless explicitly disabled return { apiBaseUrl, @@ -175,6 +180,7 @@ class BatchSnapshotOrchestrator { delayBetweenBatches, maxRetries, requestTimeout, + enableWarmUp, }; } @@ -211,7 +217,16 @@ class BatchSnapshotOrchestrator { clearTimeout(timeoutId); if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); + // Provide more specific error messages for common cold start issues + if (response.status === 405) { + throw new Error(`HTTP 405: Method Not Allowed - Possible cold start issue`); + } else if (response.status === 503) { + throw new Error(`HTTP 503: Service Unavailable - Server may be starting up`); + } else if (response.status === 502) { + throw new Error(`HTTP 502: Bad Gateway - Upstream server may be cold`); + } else { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } } const data = await response.json() as T; @@ -228,6 +243,35 @@ class BatchSnapshotOrchestrator { return new Promise(resolve => setTimeout(resolve, seconds * 1000)); } + private async warmUpApiRoute(): Promise { + console.log('🔥 Warming up API route to prevent cold start issues...'); + + try { + // Make a simple OPTIONS request to warm up the route + const url = new URL(`${this.config.apiBaseUrl}/api/v1/stats/run-snapshots-batch`); + + const response = await fetch(url.toString(), { + method: 'OPTIONS', + headers: { + 'Authorization': `Bearer ${this.config.authToken}`, + 'Content-Type': 'application/json', + }, + }); + + if (response.ok || response.status === 200) { + console.log('✅ API route warmed up successfully'); + return true; + } else { + console.log(`⚠️ API route warm-up returned status ${response.status}, but continuing...`); + return true; // Still continue as the route might be ready + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.log(`⚠️ API route warm-up failed: ${errorMessage}, but continuing...`); + return true; // Still continue as warm-up is optional + } + } + private getFriendlyErrorName(errorType: string): string { const errorMap: Record = { 'wallet_build_failed': 'Wallet Build Failed', @@ -308,9 +352,21 @@ class BatchSnapshotOrchestrator { return null; } - // For 405 errors (Method Not Allowed), wait longer as it might be a server-side issue - const waitTime = errorMessage.includes('405') ? this.config.delayBetweenBatches * 2 : this.config.delayBetweenBatches; - console.log(` ⏳ Waiting ${waitTime}s before retry...`); + // Calculate wait time with exponential backoff for cold start issues + let waitTime = this.config.delayBetweenBatches; + + if (errorMessage.includes('405') || errorMessage.includes('cold start') || errorMessage.includes('503') || errorMessage.includes('502')) { + // Cold start issue - use exponential backoff + waitTime = Math.min(this.config.delayBetweenBatches * Math.pow(2, attempt - 1), 60); + console.log(` 🥶 Cold start detected, using exponential backoff: ${waitTime}s`); + } else if (errorMessage.includes('timeout')) { + // Timeout issue - wait longer + waitTime = this.config.delayBetweenBatches * 2; + console.log(` ⏰ Timeout detected, waiting longer: ${waitTime}s`); + } else { + console.log(` ⏳ Standard retry delay: ${waitTime}s`); + } + await this.delay(waitTime); } } @@ -326,6 +382,15 @@ class BatchSnapshotOrchestrator { console.log('🔄 Starting batch snapshot orchestration...'); console.log(`📊 Configuration: batch_size=${this.config.batchSize}, delay=${this.config.delayBetweenBatches}s`); + // Warm up the API route to prevent cold start issues (if enabled) + if (this.config.enableWarmUp) { + await this.warmUpApiRoute(); + // Small delay after warm-up to ensure route is fully ready + await this.delay(2); + } else { + console.log('🔥 Warm-up disabled via ENABLE_WARM_UP=false'); + } + // First, get the total number of batches by processing batch 1 console.log('📋 Determining total batches...'); const firstBatch = await this.processBatch(1, batchId); From 14c32c059c5bc4332471aab27445487d1973b2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Sat, 11 Oct 2025 07:57:48 +0200 Subject: [PATCH 2/3] refactor(wallet): update wallet building logic to support old and new multisig wallets - Replaced the multisig wallet building function with a conditional approach that utilizes the MultisigSDK when stake keys are present. - Improved error handling for wallet building failures, providing clearer error messages. - Streamlined address generation by consolidating UTxO fetching into a single wallet address determination process. --- src/pages/api/v1/stats/run-snapshots-batch.ts | 108 ++++++++---------- 1 file changed, 46 insertions(+), 62 deletions(-) diff --git a/src/pages/api/v1/stats/run-snapshots-batch.ts b/src/pages/api/v1/stats/run-snapshots-batch.ts index 7ec9d4a7..38f7fcfd 100644 --- a/src/pages/api/v1/stats/run-snapshots-batch.ts +++ b/src/pages/api/v1/stats/run-snapshots-batch.ts @@ -1,10 +1,10 @@ import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; import type { NextApiRequest, NextApiResponse } from "next"; import { db } from "@/server/db"; -import { buildMultisigWallet } from "@/utils/common"; +import { buildWallet } from "@/utils/common"; +import { MultisigWallet, type MultisigKey } from "@/utils/multisigSDK"; import { getProvider } from "@/utils/get-provider"; -import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; -import type { UTxO, NativeScript } from "@meshsdk/core"; +import { resolvePaymentKeyHash, resolveStakeKeyHash, type UTxO } from "@meshsdk/core"; import { getBalance } from "@/utils/getBalance"; import { addressToNetwork } from "@/utils/multisigSDK"; import type { Wallet as DbWallet } from "@prisma/client"; @@ -266,69 +266,63 @@ export default async function handler( network = addressToNetwork(signerAddr); } - // Build multisig wallet for address determination - const walletData = { - id: wallet.id, - name: wallet.name, - signersAddresses: wallet.signersAddresses, - numRequiredSigners: wallet.numRequiredSigners!, - type: wallet.type || "atLeast", - stakeCredentialHash: wallet.stakeCredentialHash, - isArchived: wallet.isArchived, - description: wallet.description, - signersStakeKeys: wallet.signersStakeKeys, - signersDRepKeys: wallet.signersDRepKeys, - signersDescriptions: wallet.signersDescriptions, - clarityApiKey: wallet.clarityApiKey, - drepKey: null, - scriptType: null, - scriptCbor: wallet.scriptCbor, - verified: wallet.verified, - }; - - const mWallet = buildMultisigWallet(walletData, network); - if (!mWallet) { - console.error(`Failed to build multisig wallet for ${wallet.id.slice(0, 8)}...`); + // Build wallet conditionally: use MultisigSDK ordering if signersStakeKeys exist + let walletAddress: string; + try { + const hasStakeKeys = !!(wallet.signersStakeKeys && wallet.signersStakeKeys.length > 0); + if (hasStakeKeys) { + // Build MultisigSDK wallet with ordered keys + const keys: MultisigKey[] = []; + wallet.signersAddresses.forEach((addr: string, i: number) => { + if (!addr) return; + try { + keys.push({ keyHash: resolvePaymentKeyHash(addr), role: 0, name: wallet.signersDescriptions[i] || "" }); + } catch {} + }); + wallet.signersStakeKeys?.forEach((stakeKey: string, i: number) => { + if (!stakeKey) return; + try { + keys.push({ keyHash: resolveStakeKeyHash(stakeKey), role: 2, name: wallet.signersDescriptions[i] || "" }); + } catch {} + }); + if (keys.length === 0 && !wallet.stakeCredentialHash) { + throw new Error("No valid keys or stakeCredentialHash provided"); + } + const mWallet = new MultisigWallet( + wallet.name, + keys, + wallet.description ?? "", + wallet.numRequiredSigners ?? 1, + network, + wallet.stakeCredentialHash as undefined | string, + (wallet.type as any) || "atLeast" + ); + walletAddress = mWallet.getScript().address; + } else { + const builtWallet = buildWallet(wallet, network); + walletAddress = builtWallet.address; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown wallet build error'; + console.error(`Failed to build wallet for ${wallet.id.slice(0, 8)}...:`, errorMessage); + failures.push({ walletId: wallet.id.slice(0, 8), errorType: "wallet_build_failed", - errorMessage: "Unable to build multisig wallet from provided data", + errorMessage: "Unable to build wallet from provided data", walletStructure: getWalletStructure(wallet) }); failedInBatch++; continue; } - // Generate addresses from the built wallet - const nativeScript = { - type: wallet.type || "atLeast", - scripts: wallet.signersAddresses.map((addr: string) => ({ - type: "sig", - keyHash: resolvePaymentKeyHash(addr), - })), - }; - if (nativeScript.type == "atLeast") { - //@ts-ignore - nativeScript.required = wallet.numRequiredSigners!; - } - - const paymentAddress = serializeNativeScript( - nativeScript as NativeScript, - wallet.stakeCredentialHash as undefined | string, - network, - ).address; - - const stakeableAddress = mWallet.getScript().address; - // Determine which address to use const blockchainProvider = getProvider(network); - let paymentUtxos: UTxO[] = []; - let stakeableUtxos: UTxO[] = []; + let utxos: UTxO[] = []; try { - paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddress); - stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddress); + utxos = await blockchainProvider.fetchAddressUTxOs(walletAddress); } catch (utxoError) { const errorMessage = utxoError instanceof Error ? utxoError.message : 'Unknown UTxO fetch error'; console.error(`Failed to fetch UTxOs for wallet ${wallet.id.slice(0, 8)}...:`, errorMessage); @@ -343,16 +337,6 @@ export default async function handler( failedInBatch++; continue; } - - const paymentAddrEmpty = paymentUtxos.length === 0; - let walletAddress = paymentAddress; - - if (paymentAddrEmpty && mWallet.stakingEnabled()) { - walletAddress = stakeableAddress; - } - - // Use the UTxOs from the selected address - let utxos: UTxO[] = walletAddress === stakeableAddress ? stakeableUtxos : paymentUtxos; // If we still have no UTxOs, try the other network as fallback if (utxos.length === 0) { From 7f222bcaca2051e6e9425a73a956b015ef441013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Mon, 13 Oct 2025 07:07:27 +0200 Subject: [PATCH 3/3] fix(notification): update Discord webhook payload format for error notifications - Changed the payload key from "text" to "content" in the Discord webhook notification for daily balance snapshot failures. - Updated README documentation to reflect changes in API request structure, including removal of request body and addition of query parameters. - Enhanced wallet processing logic to include fallback to signer stake keys for network determination. - Improved documentation on batch processing features and configuration options. --- .github/workflows/daily-balance-snapshots.yml | 2 +- src/pages/api/v1/stats/README.md | 35 ++++++++++++------- src/pages/api/v1/stats/run-snapshots-batch.ts | 8 ++++- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml index e55c4414..f9227666 100644 --- a/.github/workflows/daily-balance-snapshots.yml +++ b/.github/workflows/daily-balance-snapshots.yml @@ -51,7 +51,7 @@ jobs: echo "❌ Daily balance snapshot job failed" if [ -n "${{ secrets.SNAPSHOT_ERROR_DISCORD_WEBHOOK_URL }}" ]; then curl -X POST -H 'Content-type: application/json' \ - --data "{\"text\":\"❌ Daily balance snapshots failed. Check the GitHub Actions logs.\"}" \ + --data "{\"content\":\"❌ Daily balance snapshots failed. Check the GitHub Actions logs.\"}" \ ${{ secrets.SNAPSHOT_ERROR_DISCORD_WEBHOOK_URL }} || echo "Failed to send Discord notification" else echo "SNAPSHOT_ERROR_DISCORD_WEBHOOK_URL not configured, skipping notification" diff --git a/src/pages/api/v1/stats/README.md b/src/pages/api/v1/stats/README.md index b5352195..ce8ebcc1 100644 --- a/src/pages/api/v1/stats/README.md +++ b/src/pages/api/v1/stats/README.md @@ -24,15 +24,7 @@ The endpoint requires authentication using the `SNAPSHOT_AUTH_TOKEN` environment - **Method**: POST - **Purpose**: Processes a batch of wallets for balance snapshots (main endpoint) - **Authentication**: Required (Bearer token) -- **Content-Type**: `application/json` -- **Body**: - ```json - { - "batchId": "string", - "batchNumber": number, - "batchSize": number - } - ``` +- **Parameters**: passed via query string (no request body) - **Query Parameters**: - `batchId`: Unique identifier for the batch session - `batchNumber`: Current batch number (1-based, must be ≥ 1) @@ -74,7 +66,12 @@ The endpoint requires authentication using the `SNAPSHOT_AUTH_TOKEN` environment "isArchived": boolean, "verified": number, "hasDRepKeys": boolean, - "hasClarityApiKey": boolean + "scriptCborLength": number, + "stakeCredentialLength": number, + "signersAddressesLength": number, + "signersStakeKeysLength": number, + "signersDRepKeysLength": number, + "signersDescriptionsLength": number } } ] @@ -83,6 +80,14 @@ The endpoint requires authentication using the `SNAPSHOT_AUTH_TOKEN` environment } ``` +#### Example (curl) + +```bash +curl -X POST \ + "$API_BASE_URL/api/v1/stats/run-snapshots-batch?batchId=snapshot-$(date +%s)&batchNumber=1&batchSize=5" \ + -H "Authorization: Bearer $SNAPSHOT_AUTH_TOKEN" +``` + ## Batch Processing System The new system processes wallets in small batches to avoid timeout issues: @@ -94,6 +99,8 @@ The new system processes wallets in small batches to avoid timeout issues: 4. **Fault Tolerant**: Failed batches can be retried individually 5. **Input Validation**: Comprehensive validation for batch parameters 6. **Error Tracking**: Detailed error reporting with wallet structure information +7. **Network Fallback**: If no UTxOs are found on the inferred network, the opposite network is tried +8. **Wallet Build Strategy**: Uses ordered keys via `MultisigWallet` when `signersStakeKeys` exist; otherwise falls back to legacy `buildWallet` ### Orchestrator Script The `scripts/batch-snapshot-orchestrator.ts` script manages the entire process: @@ -112,7 +119,7 @@ The `scripts/batch-snapshot-orchestrator.ts` script manages the entire process: - **`BATCH_SIZE`**: Wallets per batch (default: 5, range: 1-5) - **`DELAY_BETWEEN_BATCHES`**: Seconds between batches (default: 10) - **`MAX_RETRIES`**: Retry attempts for failed batches (default: 3) -- **`REQUEST_TIMEOUT`**: Request timeout in seconds (default: 60) +- **`REQUEST_TIMEOUT`**: Request timeout in seconds (default: 45) ## GitHub Actions Integration @@ -182,11 +189,13 @@ The orchestrator will: ### Type Safety & Validation - **Fixed Decimal Type**: Proper handling of Decimal types in database operations -- **Input Validation**: Comprehensive validation for batch parameters (batch number ≥ 1, batch size 1-100) +- **Input Validation**: Comprehensive validation for batch parameters (batch number ≥ 1, batch size 1-5) - **Error Tracking**: Enhanced error handling with detailed wallet structure information ### Configuration & Reliability - **Configurable Timeouts**: Request timeout now configurable via `REQUEST_TIMEOUT` environment variable - **Enhanced Error Handling**: UTxO fetch failures are now properly tracked and reported - **Network-Specific Reporting**: Separate tracking for mainnet and testnet wallets and balances -- **Improved Documentation**: Updated documentation to reflect all recent changes \ No newline at end of file +- **Improved Documentation**: Updated documentation to reflect all recent changes + - **Network Fallback**: Attempts the opposite network if no UTxOs are found + - **Wallet Build Logic**: Uses ordered keys with `MultisigWallet` when stake keys are available, with legacy fallback (no stake keys) \ No newline at end of file diff --git a/src/pages/api/v1/stats/run-snapshots-batch.ts b/src/pages/api/v1/stats/run-snapshots-batch.ts index 38f7fcfd..8a2f249f 100644 --- a/src/pages/api/v1/stats/run-snapshots-batch.ts +++ b/src/pages/api/v1/stats/run-snapshots-batch.ts @@ -259,11 +259,16 @@ export default async function handler( try { console.log(` Processing wallet: (${wallet.id.slice(0, 8)}...)`); - // Determine network from signer addresses + // Determine network from signer addresses, fallback to signer stake keys let network = 1; // Default to mainnet if (wallet.signersAddresses.length > 0) { const signerAddr = wallet.signersAddresses[0]!; network = addressToNetwork(signerAddr); + } else if (wallet.signersStakeKeys && wallet.signersStakeKeys.length > 0) { + const stakeAddr = wallet.signersStakeKeys.find((s) => !!s); + if (stakeAddr) { + network = addressToNetwork(stakeAddr); + } } // Build wallet conditionally: use MultisigSDK ordering if signersStakeKeys exist @@ -299,6 +304,7 @@ export default async function handler( ); walletAddress = mWallet.getScript().address; } else { + // Fallback: build the wallet without enforcing key ordering (legacy payment-script build) const builtWallet = buildWallet(wallet, network); walletAddress = builtWallet.address; }