From 5a4940a4402cc953ea459b6c6c4cebe1f8ed8d48 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Thu, 26 Feb 2026 14:31:29 +0400 Subject: [PATCH] fix: surface Yahoo rate-limit status to user, auth retry, verbose persistence - Show rate-limited message instead of generic "Failed to load" on Markets, ETF, Commodities, and Sector panels when Yahoo returns 429 - fetchYahooQuotesBatch returns rateLimited flag; early-exit after 3 misses - ETF panel skips retry loop when rate-limited, shows specific i18n message - Fallback Finnhub symbols through Yahoo when API key missing - 401-retry in runtime fetch patch for stale sidecar token after restart - diagFetch auth helper for settings window diagnostic endpoints - Verbose toggle writes to writable dataDir instead of read-only app bundle --- server/worldmonitor/market/v1/_shared.ts | 12 ++++++-- .../market/v1/get-sector-summary.ts | 2 +- .../market/v1/list-commodity-quotes.ts | 2 +- .../worldmonitor/market/v1/list-etf-flows.ts | 13 ++++++-- .../market/v1/list-market-quotes.ts | 30 ++++++++++++++----- src-tauri/Cargo.lock | 2 +- src-tauri/sidecar/local-api-server.mjs | 8 +++-- src-tauri/src/main.rs | 4 +++ src/app/data-loader.ts | 10 +++++-- src/components/ETFFlowsPanel.ts | 5 ++-- src/components/MarketPanel.ts | 4 +-- .../worldmonitor/market/v1/service_client.ts | 2 ++ .../worldmonitor/market/v1/service_server.ts | 2 ++ src/locales/en.json | 2 ++ src/services/market/index.ts | 2 ++ src/services/runtime.ts | 22 +++++++++++++- src/settings-main.ts | 21 ++++++++++--- 17 files changed, 113 insertions(+), 30 deletions(-) diff --git a/server/worldmonitor/market/v1/_shared.ts b/server/worldmonitor/market/v1/_shared.ts index 6ec65419f..736c8045a 100644 --- a/server/worldmonitor/market/v1/_shared.ts +++ b/server/worldmonitor/market/v1/_shared.ts @@ -16,13 +16,19 @@ const delay = (ms: number) => new Promise(r => setTimeout(r, ms)); export async function fetchYahooQuotesBatch( symbols: string[], -): Promise> { +): Promise<{ results: Map; rateLimited: boolean }> { const results = new Map(); + let rateLimitHits = 0; for (let i = 0; i < symbols.length; i++) { const q = await fetchYahooQuote(symbols[i]!); - if (q) results.set(symbols[i]!, q); + if (q) { + results.set(symbols[i]!, q); + } else { + rateLimitHits++; + } + if (rateLimitHits >= 3 && results.size === 0) break; } - return results; + return { results, rateLimited: rateLimitHits >= 3 && results.size === 0 }; } // Yahoo-only symbols: indices and futures not on Finnhub free tier diff --git a/server/worldmonitor/market/v1/get-sector-summary.ts b/server/worldmonitor/market/v1/get-sector-summary.ts index a657c1beb..8833c23ab 100644 --- a/server/worldmonitor/market/v1/get-sector-summary.ts +++ b/server/worldmonitor/market/v1/get-sector-summary.ts @@ -41,7 +41,7 @@ export async function getSectorSummary( if (sectors.length === 0) { const batch = await fetchYahooQuotesBatch(sectorSymbols); for (const s of sectorSymbols) { - const yahoo = batch.get(s); + const yahoo = batch.results.get(s); if (yahoo) sectors.push({ symbol: s, name: s, change: yahoo.change }); } } diff --git a/server/worldmonitor/market/v1/list-commodity-quotes.ts b/server/worldmonitor/market/v1/list-commodity-quotes.ts index 12eb0c180..0d2f19c6a 100644 --- a/server/worldmonitor/market/v1/list-commodity-quotes.ts +++ b/server/worldmonitor/market/v1/list-commodity-quotes.ts @@ -33,7 +33,7 @@ export async function listCommodityQuotes( const batch = await fetchYahooQuotesBatch(symbols); const quotes: CommodityQuote[] = []; for (const s of symbols) { - const yahoo = batch.get(s); + const yahoo = batch.results.get(s); if (yahoo) { quotes.push({ symbol: s, name: s, display: s, price: yahoo.price, change: yahoo.change, sparkline: yahoo.sparkline }); } diff --git a/server/worldmonitor/market/v1/list-etf-flows.ts b/server/worldmonitor/market/v1/list-etf-flows.ts index 4ab02a717..c78ec6afc 100644 --- a/server/worldmonitor/market/v1/list-etf-flows.ts +++ b/server/worldmonitor/market/v1/list-etf-flows.ts @@ -117,12 +117,16 @@ export async function listEtfFlows( try { const result = await cachedFetchJson(REDIS_CACHE_KEY, REDIS_CACHE_TTL, async () => { const etfs: EtfFlow[] = []; + let misses = 0; for (const etf of ETF_LIST) { const chart = await fetchEtfChart(etf.ticker); if (chart) { const parsed = parseEtfChartData(chart, etf.ticker, etf.issuer); - if (parsed) etfs.push(parsed); + if (parsed) etfs.push(parsed); else misses++; + } else { + misses++; } + if (misses >= 3 && etfs.length === 0) break; } // Stale-while-revalidate: if Yahoo rate-limited all calls, serve cached data @@ -130,7 +134,12 @@ export async function listEtfFlows( return etfCache; } - if (etfs.length === 0) return null; + if (etfs.length === 0) { + const rateLimited = misses >= 3; + return rateLimited + ? { timestamp: new Date().toISOString(), etfs: [], rateLimited: true } + : null; + } const totalVolume = etfs.reduce((sum, e) => sum + e.volume, 0); const totalEstFlow = etfs.reduce((sum, e) => sum + e.estFlow, 0); diff --git a/server/worldmonitor/market/v1/list-market-quotes.ts b/server/worldmonitor/market/v1/list-market-quotes.ts index b1f5a310a..c97ff071d 100644 --- a/server/worldmonitor/market/v1/list-market-quotes.ts +++ b/server/worldmonitor/market/v1/list-market-quotes.ts @@ -73,11 +73,20 @@ export async function listMarketQuotes( } } - // Fetch Yahoo Finance quotes for indices/futures (staggered to avoid 429) - if (yahooSymbols.length > 0) { - const batch = await fetchYahooQuotesBatch(yahooSymbols); - for (const s of yahooSymbols) { - const yahoo = batch.get(s); + // Fallback: route Finnhub symbols through Yahoo when key is missing + const missedFinnhub = apiKey + ? finnhubSymbols.filter((s) => !quotes.some((q) => q.symbol === s)) + : finnhubSymbols; + const allYahoo = [...yahooSymbols, ...missedFinnhub]; + + // Fetch Yahoo Finance quotes (staggered to avoid 429) + let yahooRateLimited = false; + if (allYahoo.length > 0) { + const batch = await fetchYahooQuotesBatch(allYahoo); + yahooRateLimited = batch.rateLimited; + for (const s of allYahoo) { + if (quotes.some((q) => q.symbol === s)) continue; + const yahoo = batch.results.get(s); if (yahoo) { quotes.push({ symbol: s, @@ -96,9 +105,16 @@ export async function listMarketQuotes( return memCached.data; } - if (quotes.length === 0) return null; + if (quotes.length === 0) { + return yahooRateLimited + ? { quotes: [], finnhubSkipped: false, skipReason: '', rateLimited: true } + : null; + } - return { quotes, finnhubSkipped: !apiKey, skipReason: !apiKey ? 'FINNHUB_API_KEY not configured' : '' }; + // Only report skipped if Finnhub key missing AND Yahoo fallback didn't cover the gap + const coveredByYahoo = finnhubSymbols.every((s) => quotes.some((q) => q.symbol === s)); + const skipped = !apiKey && !coveredByYahoo; + return { quotes, finnhubSkipped: skipped, skipReason: skipped ? 'FINNHUB_API_KEY not configured' : '' }; }); if (result?.quotes?.length) { diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index a984cf1f3..c68284842 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -4890,7 +4890,7 @@ dependencies = [ [[package]] name = "world-monitor" -version = "2.5.8" +version = "2.5.9" dependencies = [ "getrandom 0.2.17", "keyring", diff --git a/src-tauri/sidecar/local-api-server.mjs b/src-tauri/sidecar/local-api-server.mjs index 637e91351..a46af203f 100644 --- a/src-tauri/sidecar/local-api-server.mjs +++ b/src-tauri/sidecar/local-api-server.mjs @@ -396,8 +396,8 @@ const trafficLog = []; let verboseMode = false; let _verboseStatePath = null; -function loadVerboseState(resourceDir) { - _verboseStatePath = path.join(resourceDir, 'verbose-mode.json'); +function loadVerboseState(dataDir) { + _verboseStatePath = path.join(dataDir, 'verbose-mode.json'); try { const data = JSON.parse(readFileSync(_verboseStatePath, 'utf-8')); verboseMode = !!data.verboseMode; @@ -459,6 +459,7 @@ function resolveConfig(options = {}) { path.join(resourceDir, 'api'), path.join(resourceDir, '_up_', 'api'), ].find((candidate) => existsSync(candidate)) ?? path.join(resourceDir, 'api'); + const dataDir = String(options.dataDir ?? process.env.LOCAL_API_DATA_DIR ?? resourceDir); const mode = String(options.mode ?? process.env.LOCAL_API_MODE ?? 'desktop-sidecar'); const cloudFallback = String(options.cloudFallback ?? process.env.LOCAL_API_CLOUD_FALLBACK ?? '') === 'true'; const logger = options.logger ?? console; @@ -467,6 +468,7 @@ function resolveConfig(options = {}) { port, remoteBase, resourceDir, + dataDir, apiDir, mode, cloudFallback, @@ -1152,7 +1154,7 @@ async function dispatch(requestUrl, req, routes, context) { export async function createLocalApiServer(options = {}) { const context = resolveConfig(options); - loadVerboseState(context.resourceDir); + loadVerboseState(context.dataDir); const routes = await buildRouteTable(context.apiDir); const server = createServer(async (req, res) => { diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index be3da46e0..37d3e23c4 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -984,10 +984,14 @@ fn start_local_api(app: &AppHandle) -> Result<(), String> { "INFO", &format!("node args: script={script_for_node} resource_dir={resource_for_node}"), ); + let data_dir = logs_dir_path(app) + .map(|p| sanitize_path_for_node(&p)) + .unwrap_or_else(|_| resource_for_node.clone()); cmd.arg(&script_for_node) .env("LOCAL_API_PORT", DEFAULT_LOCAL_API_PORT.to_string()) .env("LOCAL_API_PORT_FILE", &port_file) .env("LOCAL_API_RESOURCE_DIR", &resource_for_node) + .env("LOCAL_API_DATA_DIR", &data_dir) .env("LOCAL_API_MODE", "tauri-sidecar") .env("LOCAL_API_TOKEN", &local_api_token) .stdout(Stdio::from(log_file)) diff --git a/src/app/data-loader.ts b/src/app/data-loader.ts index a8e610fd7..b30eb65c2 100644 --- a/src/app/data-loader.ts +++ b/src/app/data-loader.ts @@ -649,9 +649,13 @@ export class DataLoaderManager implements AppModule { const finnhubConfigMsg = 'FINNHUB_API_KEY not configured — add in Settings'; this.ctx.latestMarkets = stocksResult.data; - (this.ctx.panels['markets'] as MarketPanel).renderMarkets(stocksResult.data); + (this.ctx.panels['markets'] as MarketPanel).renderMarkets(stocksResult.data, stocksResult.rateLimited); - if (stocksResult.skipped) { + if (stocksResult.rateLimited && stocksResult.data.length === 0) { + const rlMsg = 'Market data temporarily unavailable (rate limited) — retrying shortly'; + this.ctx.panels['heatmap']?.showError(rlMsg); + this.ctx.panels['commodities']?.showError(rlMsg); + } else if (stocksResult.skipped) { this.ctx.statusPanel?.updateApi('Finnhub', { status: 'error' }); if (stocksResult.data.length === 0) { this.ctx.panels['markets']?.showConfigError(finnhubConfigMsg); @@ -678,7 +682,7 @@ export class DataLoaderManager implements AppModule { const commoditiesPanel = this.ctx.panels['commodities'] as CommoditiesPanel; const mapCommodity = (c: MarketData) => ({ display: c.display, price: c.price, change: c.change, sparkline: c.sparkline }); - let commoditiesLoaded = false; + let commoditiesLoaded = stocksResult.rateLimited && stocksResult.data.length === 0; for (let attempt = 0; attempt < 3 && !commoditiesLoaded; attempt++) { if (attempt > 0) { commoditiesPanel.showRetrying(); diff --git a/src/components/ETFFlowsPanel.ts b/src/components/ETFFlowsPanel.ts index c1f606640..cf7e8f8ee 100644 --- a/src/components/ETFFlowsPanel.ts +++ b/src/components/ETFFlowsPanel.ts @@ -53,7 +53,7 @@ export class ETFFlowsPanel extends Panel { this.data = await client.listEtfFlows({}); this.error = null; - if (this.data && this.data.etfs.length === 0 && attempt < 2) { + if (this.data && this.data.etfs.length === 0 && !this.data.rateLimited && attempt < 2) { this.showRetrying(); await new Promise(r => setTimeout(r, 20_000)); continue; @@ -86,7 +86,8 @@ export class ETFFlowsPanel extends Panel { const d = this.data; if (!d.etfs.length) { - this.setContent(`
${t('components.etfFlows.unavailable')}
`); + const msg = d.rateLimited ? t('components.etfFlows.rateLimited') : t('components.etfFlows.unavailable'); + this.setContent(`
${msg}
`); return; } diff --git a/src/components/MarketPanel.ts b/src/components/MarketPanel.ts index 88c8deaca..0a532e551 100644 --- a/src/components/MarketPanel.ts +++ b/src/components/MarketPanel.ts @@ -25,9 +25,9 @@ export class MarketPanel extends Panel { super({ id: 'markets', title: t('panels.markets') }); } - public renderMarkets(data: MarketData[]): void { + public renderMarkets(data: MarketData[], rateLimited?: boolean): void { if (data.length === 0) { - this.showError(t('common.failedMarketData')); + this.showError(rateLimited ? t('common.rateLimitedMarket') : t('common.failedMarketData')); return; } diff --git a/src/generated/client/worldmonitor/market/v1/service_client.ts b/src/generated/client/worldmonitor/market/v1/service_client.ts index 882ba0ff4..134a44e35 100644 --- a/src/generated/client/worldmonitor/market/v1/service_client.ts +++ b/src/generated/client/worldmonitor/market/v1/service_client.ts @@ -9,6 +9,7 @@ export interface ListMarketQuotesResponse { quotes: MarketQuote[]; finnhubSkipped: boolean; skipReason: string; + rateLimited?: boolean; } export interface MarketQuote { @@ -106,6 +107,7 @@ export interface ListEtfFlowsResponse { timestamp: string; summary?: EtfFlowsSummary; etfs: EtfFlow[]; + rateLimited?: boolean; } export interface EtfFlowsSummary { diff --git a/src/generated/server/worldmonitor/market/v1/service_server.ts b/src/generated/server/worldmonitor/market/v1/service_server.ts index c72c926de..213979dae 100644 --- a/src/generated/server/worldmonitor/market/v1/service_server.ts +++ b/src/generated/server/worldmonitor/market/v1/service_server.ts @@ -9,6 +9,7 @@ export interface ListMarketQuotesResponse { quotes: MarketQuote[]; finnhubSkipped: boolean; skipReason: string; + rateLimited?: boolean; } export interface MarketQuote { @@ -106,6 +107,7 @@ export interface ListEtfFlowsResponse { timestamp: string; summary?: EtfFlowsSummary; etfs: EtfFlow[]; + rateLimited?: boolean; } export interface EtfFlowsSummary { diff --git a/src/locales/en.json b/src/locales/en.json index 097ea30c0..b25a37581 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1265,6 +1265,7 @@ }, "etfFlows": { "unavailable": "ETF data temporarily unavailable", + "rateLimited": "ETF data temporarily unavailable (rate limited) — retrying shortly", "netFlow": "Net Flow", "estFlow": "Est. Flow", "totalVol": "Total Vol", @@ -1989,6 +1990,7 @@ "failedSectorData": "Failed to load sector data", "failedCommodities": "Failed to load commodities", "failedCryptoData": "Failed to load crypto data", + "rateLimitedMarket": "Market data temporarily unavailable (rate limited) — retrying shortly", "failedClusterNews": "Failed to cluster news", "noNewsAvailable": "No news available", "noActiveTechHubs": "No active tech hubs", diff --git a/src/services/market/index.ts b/src/services/market/index.ts index dd14a3b0b..b1cc695ed 100644 --- a/src/services/market/index.ts +++ b/src/services/market/index.ts @@ -55,6 +55,7 @@ export interface MarketFetchResult { data: MarketData[]; skipped?: boolean; reason?: string; + rateLimited?: boolean; } // ======================================================================== @@ -99,6 +100,7 @@ export async function fetchMultipleStocks( data, skipped: resp.finnhubSkipped || undefined, reason: resp.skipReason || undefined, + rateLimited: resp.rateLimited || undefined, }; } diff --git a/src/services/runtime.ts b/src/services/runtime.ts index dcd904d38..1dfac6016 100644 --- a/src/services/runtime.ts +++ b/src/services/runtime.ts @@ -315,8 +315,28 @@ export function installRuntimeFetchPatch(): void { try { const t0 = performance.now(); - const response = await fetchLocalWithStartupRetry(nativeFetch, localUrl, localInit); + let response = await fetchLocalWithStartupRetry(nativeFetch, localUrl, localInit); if (debug) console.log(`[fetch] ${target} → ${response.status} (${Math.round(performance.now() - t0)}ms)`); + + // Token may be stale after a sidecar restart — refresh and retry once. + if (response.status === 401 && localApiToken) { + if (debug) console.log(`[fetch] 401 from sidecar, refreshing token and retrying`); + try { + const { tryInvokeTauri } = await import('@/services/tauri-bridge'); + localApiToken = await tryInvokeTauri('get_local_api_token'); + tokenFetchedAt = Date.now(); + } catch { + localApiToken = null; + tokenFetchedAt = 0; + } + if (localApiToken) { + const retryHeaders = new Headers(init?.headers); + retryHeaders.set('Authorization', `Bearer ${localApiToken}`); + response = await fetchLocalWithStartupRetry(nativeFetch, localUrl, { ...init, headers: retryHeaders }); + if (debug) console.log(`[fetch] retry ${target} → ${response.status}`); + } + } + if (!response.ok) { if (!allowCloudFallback) { if (debug) console.log(`[fetch] local-only endpoint ${target} returned ${response.status}; skipping cloud fallback`); diff --git a/src/settings-main.ts b/src/settings-main.ts index 931f48b02..12e4d6f6d 100644 --- a/src/settings-main.ts +++ b/src/settings-main.ts @@ -177,6 +177,19 @@ function getSidecarBase(): string { return getApiBaseUrl() || 'http://127.0.0.1:46123'; } +let _diagToken: string | null = null; + +async function diagFetch(path: string, init?: RequestInit): Promise { + if (!_diagToken) { + try { + _diagToken = await tryInvokeTauri('get_local_api_token'); + } catch { /* token unavailable */ } + } + const headers = new Headers(init?.headers); + if (_diagToken) headers.set('Authorization', `Bearer ${_diagToken}`); + return fetch(`${getSidecarBase()}${path}`, { ...init, headers }); +} + function initDiagnostics(): void { const verboseToggle = document.getElementById('verboseApiLog') as HTMLInputElement | null; const fetchDebugToggle = document.getElementById('fetchDebugLog') as HTMLInputElement | null; @@ -196,7 +209,7 @@ function initDiagnostics(): void { async function syncVerboseState(): Promise { if (!verboseToggle) return; try { - const res = await fetch(`${getSidecarBase()}/api/local-debug-toggle`); + const res = await diagFetch('/api/local-debug-toggle'); const data = await res.json(); verboseToggle.checked = data.verboseMode; } catch { /* sidecar not running */ } @@ -204,7 +217,7 @@ function initDiagnostics(): void { verboseToggle?.addEventListener('change', async () => { try { - const res = await fetch(`${getSidecarBase()}/api/local-debug-toggle`, { method: 'POST' }); + const res = await diagFetch('/api/local-debug-toggle', { method: 'POST' }); const data = await res.json(); if (verboseToggle) verboseToggle.checked = data.verboseMode; setActionStatus(data.verboseMode ? t('modals.settingsWindow.verboseOn') : t('modals.settingsWindow.verboseOff'), 'ok'); @@ -218,7 +231,7 @@ function initDiagnostics(): void { async function refreshTrafficLog(): Promise { if (!trafficLogEl) return; try { - const res = await fetch(`${getSidecarBase()}/api/local-traffic-log`); + const res = await diagFetch('/api/local-traffic-log'); const data = await res.json(); const entries: Array<{ timestamp: string; method: string; path: string; status: number; durationMs: number }> = data.entries || []; if (trafficCount) trafficCount.textContent = `(${entries.length})`; @@ -244,7 +257,7 @@ function initDiagnostics(): void { clearBtn?.addEventListener('click', async () => { try { - await fetch(`${getSidecarBase()}/api/local-traffic-log`, { method: 'DELETE' }); + await diagFetch('/api/local-traffic-log', { method: 'DELETE' }); } catch { /* ignore */ } if (trafficLogEl) trafficLogEl.innerHTML = `

${t('modals.settingsWindow.logCleared')}

`; if (trafficCount) trafficCount.textContent = '(0)';