From c23d70cedc33db79222c80646a2c2deee53152e1 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 30 Jul 2024 00:12:56 -0600 Subject: [PATCH 1/3] feat: add historical state regen (#6033) * feat: add historical state regen * chore: wire up metrics * chore: make historical state regen module optional * chore: persist pubkey cache across historical state regen runs * chore: cleanup worker termination * chore: fix worker usage * fix: swap Level for ClassicLevel for multithreading * fix: getStateV2 state handling hack * chore: update classic-level * chore: fix build errors * chore: add comments * chore: fix test worker path * chore: simplify function naming * chore: optimize getSlotFromOffset * chore: refactor to avoid needless deserialization * fix: update metrics names * feat: add historical state regen dashboard * fix: update vm dashboards with historical state worker * chore: fix test data * feat: transfer state across worker boundary * chore: address some pr comments * chore: clean module close * feat: add metrics --------- Co-authored-by: Matthew Keil Co-authored-by: Tuyen Nguyen --- .../lodestar_historical_state_regen.json | 2646 +++++++++++++++++ dashboards/lodestar_vm_host.json | 318 +- .../src/api/impl/beacon/state/utils.ts | 2 +- packages/beacon-node/src/chain/chain.ts | 87 +- .../historicalState/getHistoricalState.ts | 111 + .../src/chain/historicalState/index.ts | 67 + .../src/chain/historicalState/types.ts | 54 + .../src/chain/historicalState/worker.ts | 231 ++ packages/beacon-node/src/chain/interface.ts | 4 + .../beacon-node/src/metrics/server/http.ts | 7 +- packages/beacon-node/src/node/nodejs.ts | 15 +- packages/db/src/controller/level.ts | 7 +- 12 files changed, 3461 insertions(+), 88 deletions(-) create mode 100644 dashboards/lodestar_historical_state_regen.json create mode 100644 packages/beacon-node/src/chain/historicalState/getHistoricalState.ts create mode 100644 packages/beacon-node/src/chain/historicalState/index.ts create mode 100644 packages/beacon-node/src/chain/historicalState/types.ts create mode 100644 packages/beacon-node/src/chain/historicalState/worker.ts diff --git a/dashboards/lodestar_historical_state_regen.json b/dashboards/lodestar_historical_state_regen.json new file mode 100644 index 000000000000..20eddcd1f31e --- /dev/null +++ b/dashboards/lodestar_historical_state_regen.json @@ -0,0 +1,2646 @@ +{ + "__inputs": [ + { + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + }, + { + "description": "", + "label": "Beacon node job name", + "name": "VAR_BEACON_JOB", + "type": "constant", + "value": "beacon" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "lodestar" + ], + "targetBlank": false, + "title": "Lodestar dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 535, + "panels": [], + "title": "VM", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 536, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "lodestar_historical_state_worker_nodejs_heap_space_size_used_bytes{job=~\"$beacon_job|beacon\"}", + "interval": "", + "legendFormat": "{{space}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "lodestar_historical_state_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"}", + "hide": false, + "interval": "", + "legendFormat": "external_memory", + "range": true, + "refId": "B" + } + ], + "title": "Heap Allocations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GC Time", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "GC Bytes" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 537, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0-beta1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_worker_nodejs_gc_pause_seconds_total{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "hide": false, + "legendFormat": "{{gctype}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_worker_nodejs_gc_reclaimed_bytes_total{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "format": "time_series", + "hide": false, + "legendFormat": "{{gctype}}", + "range": true, + "refId": "C" + } + ], + "title": "GC pause time rate + reclaimed bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "set_immediate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "min" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 538, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(lodestar_historical_state_worker_nodejs_eventloop_lag_min_seconds{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "hide": false, + "legendFormat": "min", + "range": true, + "refId": "0" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(lodestar_historical_state_worker_nodejs_eventloop_lag_p50_seconds{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(lodestar_historical_state_worker_nodejs_eventloop_lag_p90_seconds{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(lodestar_historical_state_worker_nodejs_eventloop_lag_p99_seconds{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(lodestar_historical_state_worker_nodejs_eventloop_lag_max_seconds{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "hide": false, + "legendFormat": "max", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(lodestar_historical_state_worker_nodejs_eventloop_lag_seconds{job=~\"$beacon_job|beacon\"}[$rate_interval])", + "hide": false, + "legendFormat": "set_immediate", + "range": true, + "refId": "E" + } + ], + "title": "Event loop lag", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 25, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Job Queue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0-beta1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_queue_job_time_seconds_sum[$rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "block_processor", + "refId": "A" + } + ], + "title": "Utilization rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 81, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "12*rate(lodestar_historical_state_queue_job_time_seconds_count[6m])", + "instant": false, + "interval": "", + "legendFormat": "block_processor", + "refId": "A" + } + ], + "title": "Jobs / slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 100, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_queue_job_time_seconds_sum[$rate_interval])/rate(lodestar_historical_state_queue_job_time_seconds_count[$rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "block_processor", + "refId": "A" + } + ], + "title": "Job time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 128, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "lodestar_historical_state_queue_length", + "instant": false, + "interval": "", + "legendFormat": "block_processor", + "refId": "A" + } + ], + "title": "Queue length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 127, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_queue_job_wait_time_seconds_sum[$rate_interval])/rate(lodestar_historical_state_queue_job_wait_time_seconds_count[$rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "block_processor", + "refId": "A" + } + ], + "title": "Job wait time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 126, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_queue_dropped_jobs_total[$rate_interval])/(rate(lodestar_historical_state_queue_job_time_seconds_count[$rate_interval])+rate(lodestar_historical_state_queue_dropped_jobs_total[$rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "block_processor", + "refId": "A" + } + ], + "title": "Dropped jobs %", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 108, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Beacon state transition", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 528, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Magma", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "32*12*rate(lodestar_historical_state_stfn_epoch_transition_seconds_bucket[$rate_interval])", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Epoch transition time", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 529, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Magma", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "12*rate(lodestar_historical_state_stfn_process_block_seconds_bucket[$rate_interval])", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Process block time", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 120, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(lodestar_historical_state_stfn_epoch_transition_seconds_sum[$rate_interval])\n/\nrate(lodestar_historical_state_stfn_epoch_transition_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "epoch transition", + "range": true, + "refId": "A" + } + ], + "title": "Epoch transition avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 121, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_process_block_seconds_sum[$rate_interval])\n/\nrate(lodestar_historical_state_stfn_process_block_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "process block time", + "range": true, + "refId": "A" + } + ], + "title": "Process block avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 534, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_epoch_transition_step_seconds_sum[$rate_interval])\n/\nrate(lodestar_historical_state_stfn_epoch_transition_step_seconds_count[$rate_interval])", + "instant": false, + "legendFormat": "{{step}}", + "range": true, + "refId": "A" + } + ], + "title": "Epoch Transition By Steps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 525, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_process_block_commit_seconds_sum[$rate_interval])\n/\nrate(lodestar_historical_state_stfn_process_block_commit_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "process block time", + "range": true, + "refId": "A" + } + ], + "title": "Process block commit step avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 524, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(lodestar_historical_state_stfn_epoch_transition_commit_seconds_sum[$rate_interval])\n/\nrate(lodestar_historical_state_stfn_epoch_transition_commit_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "epoch transition", + "range": true, + "refId": "A" + } + ], + "title": "Epoch transition commit step avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "process block time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 123, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(lodestar_historical_state_stfn_process_block_seconds_sum[6m])", + "interval": "", + "legendFormat": "process block time", + "range": true, + "refId": "A" + } + ], + "title": "Process block utilization rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "process block time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 122, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(lodestar_historical_state_stfn_epoch_transition_seconds_sum[13m])", + "interval": "", + "legendFormat": "process block time", + "range": true, + "refId": "A" + } + ], + "title": "Epoch transition utilization rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "process block time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 125, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "12*rate(lodestar_historical_state_stfn_process_block_seconds_count[6m])", + "interval": "", + "legendFormat": "process block time", + "range": true, + "refId": "A" + } + ], + "title": "process block / slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "number of epoch transition" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 124, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "384 * rate(lodestar_historical_state_stfn_epoch_transition_seconds_count[13m])", + "interval": "", + "legendFormat": "number of epoch transition", + "range": true, + "refId": "A" + } + ], + "title": "Epoch transitions / epoch", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 527, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_validators_nodes_populated_miss_total{source=\"stateTransition\"} [$rate_interval])\n/on(instance)\nrate(lodestar_historical_state_stfn_state_cloned_count_count[$rate_interval])", + "interval": "", + "legendFormat": "stateTransition-validators", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_balances_nodes_populated_miss_total{source=\"stateTransition\"} [$rate_interval])\n/on(instance)\nrate(lodestar_historical_state_stfn_state_cloned_count_count[$rate_interval])", + "hide": false, + "legendFormat": "stateTransition-balances", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_validators_nodes_populated_miss_total{source=\"processSlots\"} [$rate_interval])\n/on(instance)\nrate(lodestar_historical_state_stfn_state_cloned_count_count[$rate_interval])", + "hide": false, + "legendFormat": "processSlots-validators", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_balances_nodes_populated_miss_total{source=\"processSlots\"} [$rate_interval])\n/on(instance)\nrate(lodestar_historical_state_stfn_state_cloned_count_count[$rate_interval])", + "hide": false, + "legendFormat": "processSlots-balances", + "range": true, + "refId": "D" + } + ], + "title": "State SSZ cache miss rate on preState", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 526, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_hash_tree_root_seconds_sum[$rate_interval])\n/ on(source)\nrate(lodestar_historical_state_stfn_hash_tree_root_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "State hash_tree_root avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 523, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Magma", + "steps": 50 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_state_cloned_count_bucket[$rate_interval])", + "format": "heatmap", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Clone count per state", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 521, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_balances_nodes_populated_miss_total{source=\"processSlots\"} [$rate_interval])\n/\nrate(lodestar_historical_state_stfn_state_clone_total[$rate_interval])", + "legendFormat": "balances-{{source}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_balances_nodes_populated_miss_total{source=\"stateTransition\"} [$rate_interval])\n/\nrate(lodestar_historical_state_stfn_state_clone_total[$rate_interval])", + "hide": false, + "legendFormat": "balances-{{source}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_validators_nodes_populated_miss_total{source=\"processSlots\"} [$rate_interval])\n/\nrate(lodestar_historical_state_stfn_state_clone_total[$rate_interval])", + "hide": false, + "legendFormat": "validators-{{source}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_stfn_validators_nodes_populated_miss_total{source=\"stateTransition\"} [$rate_interval])\n/\nrate(lodestar_historical_state_stfn_state_clone_total[$rate_interval])", + "hide": false, + "legendFormat": "validators-{{source}}", + "range": true, + "refId": "D" + } + ], + "title": "State transition fn nodes populated cache miss", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "lodestar" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1h", + "value": "1h" + }, + "hide": 0, + "label": "rate() interval", + "name": "rate_interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_rate_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_local" + }, + "filters": [ + { + "condition": "", + "key": "instance", + "operator": "=", + "value": "unstable-lg1k-hzax41" + } + ], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "description": "Job name used in Prometheus config to scrape Beacon node", + "hide": 2, + "label": "Beacon node job name", + "name": "beacon_job", + "query": "${VAR_BEACON_JOB}", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Lodestar - historical state regen", + "uid": "lodestar_historical_state_regen", + "version": 32, + "weekStart": "monday" +} diff --git a/dashboards/lodestar_vm_host.json b/dashboards/lodestar_vm_host.json index 3f6c18bc19c1..457d4d120fa4 100644 --- a/dashboards/lodestar_vm_host.json +++ b/dashboards/lodestar_vm_host.json @@ -177,7 +177,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes) + sum(network_worker_nodejs_heap_size_total_bytes))\nor\n(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes))", + "expr": "(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes) + sum(network_worker_nodejs_heap_size_total_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_total_bytes))\nor\n(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_total_bytes))", "hide": false, "interval": "", "legendFormat": "node allocated heap", @@ -191,7 +191,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes) + sum(network_worker_nodejs_heap_size_used_bytes)) \nor\n(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes)) ", + "expr": "(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes) + sum(network_worker_nodejs_heap_size_used_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_used_bytes)) \nor\n(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_used_bytes)) ", "hide": false, "interval": "", "legendFormat": "node used heap", @@ -205,7 +205,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes) + sum(network_worker_nodejs_external_memory_bytes))\nor\n(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes))", + "expr": "(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes) + sum(network_worker_nodejs_external_memory_bytes) + sum(lodestar_historical_state_worker_nodejs_external_memory_bytes))\nor\n(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes) + sum(lodestar_historical_state_worker_nodejs_external_memory_bytes))", "hide": false, "interval": "", "legendFormat": "node external memory", @@ -291,7 +291,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (space) ({__name__=~\"nodejs_heap_space_size_used_bytes|network_worker_nodejs_heap_space_size_used_bytes|discv5_worker_nodejs_heap_space_size_used_bytes\"})", + "expr": "sum by (space) ({__name__=~\"nodejs_heap_space_size_used_bytes|network_worker_nodejs_heap_space_size_used_bytes|discv5_worker_nodejs_heap_space_size_used_bytes|lodestar_historical_state_worker_nodejs_heap_space_size_used_bytes\")", "hide": false, "interval": "", "legendFormat": "{{space}}", @@ -305,7 +305,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + network_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})\nor\n(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})", + "expr": "(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + network_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + lodestar_historical_state_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})\nor\n(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + lodestar_historical_state_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})", "hide": false, "interval": "", "legendFormat": "external_memory", @@ -786,6 +786,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -883,6 +884,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1000,6 +1002,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1027,6 +1030,222 @@ "x": 0, "y": 34 }, + "id": 562, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "lodestar_historical_state_worker_nodejs_heap_space_size_used_bytes", + "interval": "", + "legendFormat": "{{space}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "lodestar_historical_state_worker_nodejs_external_memory_bytes", + "hide": false, + "interval": "", + "legendFormat": "external_memory", + "range": true, + "refId": "B" + } + ], + "title": "Historical State Worker Thread - Heap Allocations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GC Time", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "GC Bytes" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 563, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0-beta1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_worker_nodejs_gc_pause_seconds_total[$rate_interval])", + "hide": false, + "legendFormat": "{{gctype}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_worker_nodejs_gc_reclaimed_bytes_total[$rate_interval])", + "format": "time_series", + "hide": false, + "legendFormat": "{{gctype}}", + "range": true, + "refId": "C" + } + ], + "title": "HIstorical State Worker Thread - GC pause time rate + reclaimed bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, "id": 542, "options": { "graph": {}, @@ -1097,6 +1316,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1143,7 +1363,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 42 }, "id": 548, "options": { @@ -1199,7 +1419,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 50 }, "id": 12, "panels": [], @@ -1239,6 +1459,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1368,7 +1589,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 51 }, "id": 555, "options": { @@ -1483,6 +1704,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1612,7 +1834,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 51 }, "id": 559, "options": { @@ -1856,7 +2078,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 560, "options": { @@ -2100,7 +2322,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 561, "options": { @@ -2241,7 +2463,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 6, "options": { @@ -2364,7 +2586,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 67 }, "id": 42, "options": { @@ -2443,7 +2665,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 75 }, "id": 268, "options": { @@ -2484,7 +2706,7 @@ "h": 1, "w": 24, "x": 0, - "y": 75 + "y": 83 }, "id": 104, "panels": [], @@ -2550,7 +2772,7 @@ "h": 6, "w": 12, "x": 0, - "y": 76 + "y": 84 }, "id": 102, "options": { @@ -2659,7 +2881,7 @@ "h": 6, "w": 12, "x": 12, - "y": 76 + "y": 84 }, "id": 172, "options": { @@ -2744,7 +2966,7 @@ "h": 6, "w": 12, "x": 0, - "y": 82 + "y": 90 }, "id": 171, "options": { @@ -2787,7 +3009,7 @@ "h": 6, "w": 12, "x": 12, - "y": 82 + "y": 90 }, "id": 99, "options": { @@ -2828,7 +3050,7 @@ "h": 1, "w": 24, "x": 0, - "y": 88 + "y": 96 }, "id": 367, "panels": [], @@ -2894,7 +3116,7 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 97 }, "id": 353, "options": { @@ -2977,7 +3199,7 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 97 }, "id": 355, "options": { @@ -3059,7 +3281,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 105 }, "id": 357, "options": { @@ -3522,7 +3744,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 105 }, "id": 359, "links": [], @@ -3665,7 +3887,7 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 113 }, "id": 361, "options": { @@ -3774,7 +3996,7 @@ "h": 8, "w": 12, "x": 12, - "y": 105 + "y": 113 }, "id": 363, "options": { @@ -3866,7 +4088,7 @@ "h": 8, "w": 12, "x": 0, - "y": 113 + "y": 121 }, "id": 365, "options": { @@ -3919,7 +4141,7 @@ "h": 1, "w": 24, "x": 0, - "y": 121 + "y": 129 }, "id": 46, "panels": [], @@ -3985,7 +4207,7 @@ "h": 8, "w": 12, "x": 0, - "y": 122 + "y": 130 }, "id": 48, "options": { @@ -4112,7 +4334,7 @@ "h": 8, "w": 12, "x": 12, - "y": 122 + "y": 130 }, "id": 50, "options": { @@ -4155,7 +4377,7 @@ "h": 1, "w": 24, "x": 0, - "y": 130 + "y": 138 }, "id": 527, "panels": [], @@ -4221,7 +4443,7 @@ "h": 8, "w": 12, "x": 0, - "y": 131 + "y": 139 }, "id": 529, "options": { @@ -4366,7 +4588,7 @@ "h": 8, "w": 12, "x": 12, - "y": 131 + "y": 139 }, "id": 531, "options": { @@ -4410,7 +4632,7 @@ "h": 1, "w": 24, "x": 0, - "y": 139 + "y": 147 }, "id": 86, "panels": [], @@ -4475,7 +4697,7 @@ "h": 11, "w": 12, "x": 0, - "y": 140 + "y": 148 }, "id": 84, "options": { @@ -4558,7 +4780,7 @@ "h": 11, "w": 12, "x": 12, - "y": 140 + "y": 148 }, "id": 87, "options": { @@ -4639,7 +4861,7 @@ "h": 8, "w": 12, "x": 0, - "y": 151 + "y": 159 }, "id": 516, "options": { @@ -4730,7 +4952,7 @@ "h": 8, "w": 12, "x": 12, - "y": 151 + "y": 159 }, "id": 515, "options": { @@ -4809,7 +5031,7 @@ "h": 8, "w": 12, "x": 0, - "y": 159 + "y": 167 }, "id": 513, "options": { @@ -4888,7 +5110,7 @@ "h": 8, "w": 12, "x": 12, - "y": 159 + "y": 167 }, "id": 517, "options": { @@ -4928,7 +5150,7 @@ "h": 1, "w": 24, "x": 0, - "y": 167 + "y": 175 }, "id": 164, "panels": [], @@ -4995,7 +5217,7 @@ "h": 9, "w": 12, "x": 0, - "y": 168 + "y": 176 }, "id": 160, "options": { @@ -5079,7 +5301,7 @@ "h": 9, "w": 12, "x": 12, - "y": 168 + "y": 176 }, "id": 162, "options": { @@ -5163,7 +5385,7 @@ "h": 9, "w": 12, "x": 0, - "y": 177 + "y": 185 }, "id": 481, "options": { @@ -5247,7 +5469,7 @@ "h": 9, "w": 12, "x": 12, - "y": 177 + "y": 185 }, "id": 480, "options": { @@ -5354,7 +5576,7 @@ "h": 8, "w": 12, "x": 0, - "y": 186 + "y": 194 }, "id": 535, "options": { @@ -5459,7 +5681,7 @@ "h": 8, "w": 12, "x": 12, - "y": 186 + "y": 194 }, "id": 537, "options": { @@ -5540,7 +5762,7 @@ "h": 8, "w": 12, "x": 0, - "y": 194 + "y": 202 }, "id": 533, "options": { diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index a2079afaf9b8..7bea4fec7071 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -66,7 +66,7 @@ export async function getStateResponseWithRegen( ? await chain.getStateByStateRoot(rootOrSlot, {allowRegen: true}) : rootOrSlot >= chain.forkChoice.getFinalizedBlock().slot ? await chain.getStateBySlot(rootOrSlot, {allowRegen: true}) - : null; // TODO implement historical state regen + : await chain.getHistoricalStateBySlot(rootOrSlot); if (!res) { throw new ApiError(404, `No state found for id '${stateId}'`); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 081de6f85063..f32a44bf4e5d 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -89,6 +89,7 @@ import {BlockAttributes, produceBlockBody, produceCommonBlockBody} from "./produ import {computeNewStateRoot} from "./produceBlock/computeNewStateRoot.js"; import {BlockInput} from "./blocks/types.js"; import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; +import {HistoricalStateRegen} from "./historicalState/index.js"; import {BlockRewards, computeBlockRewards} from "./rewards/blockRewards.js"; import {ShufflingCache} from "./shufflingCache.js"; import {BlockStateCacheImpl} from "./stateCache/blockStateCacheImpl.js"; @@ -128,6 +129,7 @@ export class BeaconChain implements IBeaconChain { readonly regen: QueuedStateRegenerator; readonly lightClientServer?: LightClientServer; readonly reprocessController: ReprocessController; + readonly historicalStateRegen?: HistoricalStateRegen; // Ops pool readonly attestationPool: AttestationPool; @@ -185,6 +187,7 @@ export class BeaconChain implements IBeaconChain { eth1, executionEngine, executionBuilder, + historicalStateRegen, }: { config: BeaconConfig; db: IBeaconDb; @@ -197,6 +200,7 @@ export class BeaconChain implements IBeaconChain { eth1: IEth1ForBlockProduction; executionEngine: IExecutionEngine; executionBuilder?: IExecutionBuilder; + historicalStateRegen?: HistoricalStateRegen; } ) { this.opts = opts; @@ -211,6 +215,7 @@ export class BeaconChain implements IBeaconChain { this.eth1 = eth1; this.executionEngine = executionEngine; this.executionBuilder = executionBuilder; + this.historicalStateRegen = historicalStateRegen; const signal = this.abortController.signal; const emitter = new ChainEventEmitter(); // by default, verify signatures on both main threads and worker threads @@ -418,47 +423,61 @@ export class BeaconChain implements IBeaconChain { ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null> { const finalizedBlock = this.forkChoice.getFinalizedBlock(); - if (slot >= finalizedBlock.slot) { - // request for non-finalized state - - if (opts?.allowRegen) { - // Find closest canonical block to slot, then trigger regen - const block = this.forkChoice.getCanonicalBlockClosestLteSlot(slot) ?? finalizedBlock; - const state = await this.regen.getBlockSlotState( - block.blockRoot, - slot, - {dontTransferCache: true}, - RegenCaller.restApi - ); - return { + if (slot < finalizedBlock.slot) { + // request for finalized state not supported in this API + // fall back to caller to look in db or getHistoricalStateBySlot + return null; + } + + if (opts?.allowRegen) { + // Find closest canonical block to slot, then trigger regen + const block = this.forkChoice.getCanonicalBlockClosestLteSlot(slot) ?? finalizedBlock; + const state = await this.regen.getBlockSlotState( + block.blockRoot, + slot, + {dontTransferCache: true}, + RegenCaller.restApi + ); + return { + state, + executionOptimistic: isOptimisticBlock(block), + finalized: slot === finalizedBlock.slot && finalizedBlock.slot !== GENESIS_SLOT, + }; + } else { + // Just check if state is already in the cache. If it's not dialed to the correct slot, + // do not bother in advancing the state. restApiCanTriggerRegen == false means do no work + const block = this.forkChoice.getCanonicalBlockAtSlot(slot); + if (!block) { + return null; + } + + const state = this.regen.getStateSync(block.stateRoot); + return ( + state && { state, executionOptimistic: isOptimisticBlock(block), finalized: slot === finalizedBlock.slot && finalizedBlock.slot !== GENESIS_SLOT, - }; - } else { - // Just check if state is already in the cache. If it's not dialed to the correct slot, - // do not bother in advancing the state. restApiCanTriggerRegen == false means do no work - const block = this.forkChoice.getCanonicalBlockAtSlot(slot); - if (!block) { - return null; } + ); + } + } - const state = this.regen.getStateSync(block.stateRoot); - return ( - state && { - state, - executionOptimistic: isOptimisticBlock(block), - finalized: slot === finalizedBlock.slot && finalizedBlock.slot !== GENESIS_SLOT, - } - ); - } - } else { - // request for finalized state + async getHistoricalStateBySlot( + slot: number + ): Promise<{state: Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> { + const finalizedBlock = this.forkChoice.getFinalizedBlock(); + + if (slot >= finalizedBlock.slot) { + return null; + } - // do not attempt regen, just check if state is already in DB - const state = await this.db.stateArchive.get(slot); - return state && {state, executionOptimistic: false, finalized: true}; + // request for finalized state using historical state regen + const stateSerialized = await this.historicalStateRegen?.getHistoricalState(slot); + if (!stateSerialized) { + return null; } + + return {state: stateSerialized, executionOptimistic: false, finalized: true}; } async getStateByStateRoot( diff --git a/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts b/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts new file mode 100644 index 000000000000..ada4f3c284d7 --- /dev/null +++ b/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts @@ -0,0 +1,111 @@ +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + DataAvailableStatus, + ExecutionPayloadStatus, + PubkeyIndexMap, + createCachedBeaconState, + stateTransition, +} from "@lodestar/state-transition"; +import {BeaconConfig} from "@lodestar/config"; +import {IBeaconDb} from "../../db/index.js"; +import {HistoricalStateRegenMetrics, RegenErrorType} from "./types.js"; + +/** + * Populate a PubkeyIndexMap with any new entries based on a BeaconState + */ +export function syncPubkeyCache(state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap): void { + // Get the validators sub tree once for all the loop + const validators = state.validators; + + const newCount = state.validators.length; + for (let i = pubkey2index.size; i < newCount; i++) { + const pubkey = validators.getReadonly(i).pubkey; + pubkey2index.set(pubkey, i); + } +} + +/** + * Get the nearest BeaconState at or before a slot + */ +export async function getNearestState( + slot: number, + config: BeaconConfig, + db: IBeaconDb, + pubkey2index: PubkeyIndexMap +): Promise { + const states = await db.stateArchive.values({limit: 1, lte: slot, reverse: true}); + if (!states.length) { + throw new Error("No near state found in the database"); + } + + const state = states[0]; + syncPubkeyCache(state, pubkey2index); + + return createCachedBeaconState( + state, + { + config, + pubkey2index, + index2pubkey: [], + }, + { + skipSyncPubkeys: true, + } + ); +} + +/** + * Get and regenerate a historical state + */ +export async function getHistoricalState( + slot: number, + config: BeaconConfig, + db: IBeaconDb, + pubkey2index: PubkeyIndexMap, + metrics?: HistoricalStateRegenMetrics +): Promise { + const regenTimer = metrics?.regenTime.startTimer(); + + const loadStateTimer = metrics?.loadStateTime.startTimer(); + let state = await getNearestState(slot, config, db, pubkey2index).catch((e) => { + metrics?.regenErrorCount.inc({reason: RegenErrorType.loadState}); + throw e; + }); + loadStateTimer?.(); + + const transitionTimer = metrics?.stateTransitionTime.startTimer(); + let blockCount = 0; + for await (const block of db.blockArchive.valuesStream({gt: state.slot, lte: slot})) { + try { + state = stateTransition( + state, + block, + { + verifyProposer: false, + verifySignatures: false, + verifyStateRoot: false, + executionPayloadStatus: ExecutionPayloadStatus.valid, + dataAvailableStatus: DataAvailableStatus.available, + }, + metrics + ); + } catch (e) { + metrics?.regenErrorCount.inc({reason: RegenErrorType.blockProcessing}); + throw e; + } + blockCount++; + if (Buffer.compare(state.hashTreeRoot(), block.message.stateRoot) !== 0) { + metrics?.regenErrorCount.inc({reason: RegenErrorType.invalidStateRoot}); + } + } + metrics?.stateTransitionBlocks.observe(blockCount); + transitionTimer?.(); + + const serializeTimer = metrics?.stateSerializationTime.startTimer(); + const stateBytes = state.serialize(); + serializeTimer?.(); + + regenTimer?.(); + return stateBytes; +} diff --git a/packages/beacon-node/src/chain/historicalState/index.ts b/packages/beacon-node/src/chain/historicalState/index.ts new file mode 100644 index 000000000000..8688bcf9f372 --- /dev/null +++ b/packages/beacon-node/src/chain/historicalState/index.ts @@ -0,0 +1,67 @@ +import path from "node:path"; +import {ModuleThread, Thread, spawn, Worker} from "@chainsafe/threads"; +import {chainConfigToJson} from "@lodestar/config"; +import {LoggerNode} from "@lodestar/logger/node"; +import { + HistoricalStateRegenInitModules, + HistoricalStateRegenModules, + HistoricalStateWorkerApi, + HistoricalStateWorkerData, +} from "./types.js"; + +// Worker constructor consider the path relative to the current working directory +const WORKER_DIR = process.env.NODE_ENV === "test" ? "../../../lib/chain/historicalState" : "./"; + +/** + * HistoricalStateRegen limits the damage from recreating historical states + * by running regen in a separate worker thread. + */ +export class HistoricalStateRegen implements HistoricalStateWorkerApi { + private readonly api: ModuleThread; + private readonly logger: LoggerNode; + + constructor(modules: HistoricalStateRegenModules) { + this.api = modules.api; + this.logger = modules.logger; + modules.signal?.addEventListener("abort", () => this.close(), {once: true}); + } + static async init(modules: HistoricalStateRegenInitModules): Promise { + const workerData: HistoricalStateWorkerData = { + chainConfigJson: chainConfigToJson(modules.config), + genesisValidatorsRoot: modules.config.genesisValidatorsRoot, + genesisTime: modules.opts.genesisTime, + maxConcurrency: 1, + maxLength: 50, + dbLocation: modules.opts.dbLocation, + metricsEnabled: Boolean(modules.metrics), + loggerOpts: modules.logger.toOpts(), + }; + + const worker = new Worker(path.join(WORKER_DIR, "worker.js"), { + workerData, + } as ConstructorParameters[1]); + + const api = await spawn(worker, { + // A Lodestar Node may do very expensive task at start blocking the event loop and causing + // the initialization to timeout. The number below is big enough to almost disable the timeout + timeout: 5 * 60 * 1000, + }); + + return new HistoricalStateRegen({...modules, api}); + } + + async scrapeMetrics(): Promise { + return this.api.scrapeMetrics(); + } + + async close(): Promise { + await this.api.close(); + this.logger.debug("Terminating historical state worker"); + await Thread.terminate(this.api); + this.logger.debug("Terminated historical state worker"); + } + + async getHistoricalState(slot: number): Promise { + return this.api.getHistoricalState(slot); + } +} diff --git a/packages/beacon-node/src/chain/historicalState/types.ts b/packages/beacon-node/src/chain/historicalState/types.ts new file mode 100644 index 000000000000..5bc813141a5d --- /dev/null +++ b/packages/beacon-node/src/chain/historicalState/types.ts @@ -0,0 +1,54 @@ +import {ModuleThread} from "@chainsafe/threads"; +import {BeaconConfig} from "@lodestar/config"; +import {LoggerNode, LoggerNodeOpts} from "@lodestar/logger/node"; +import {BeaconStateTransitionMetrics} from "@lodestar/state-transition"; +import {Gauge, Histogram} from "@lodestar/utils"; +import {Metrics} from "../../metrics/index.js"; + +export type HistoricalStateRegenInitModules = { + opts: { + genesisTime: number; + dbLocation: string; + }; + config: BeaconConfig; + logger: LoggerNode; + metrics: Metrics | null; + signal?: AbortSignal; +}; +export type HistoricalStateRegenModules = HistoricalStateRegenInitModules & { + api: ModuleThread; +}; + +export type HistoricalStateWorkerData = { + chainConfigJson: Record; + genesisValidatorsRoot: Uint8Array; + genesisTime: number; + maxConcurrency: number; + maxLength: number; + dbLocation: string; + metricsEnabled: boolean; + loggerOpts: LoggerNodeOpts; +}; + +export type HistoricalStateWorkerApi = { + close(): Promise; + scrapeMetrics(): Promise; + getHistoricalState(slot: number): Promise; +}; + +export enum RegenErrorType { + loadState = "load_state", + invalidStateRoot = "invalid_state_root", + blockProcessing = "block_processing", +} + +export type HistoricalStateRegenMetrics = BeaconStateTransitionMetrics & { + regenTime: Histogram; + loadStateTime: Histogram; + stateTransitionTime: Histogram; + stateTransitionBlocks: Histogram; + stateSerializationTime: Histogram; + regenRequestCount: Gauge; + regenSuccessCount: Gauge; + regenErrorCount: Gauge<{reason: RegenErrorType}>; +}; diff --git a/packages/beacon-node/src/chain/historicalState/worker.ts b/packages/beacon-node/src/chain/historicalState/worker.ts new file mode 100644 index 000000000000..9a9f9cc9cd0e --- /dev/null +++ b/packages/beacon-node/src/chain/historicalState/worker.ts @@ -0,0 +1,231 @@ +import worker from "node:worker_threads"; +import {Transfer, expose} from "@chainsafe/threads/worker"; +import {createBeaconConfig, chainConfigFromJson} from "@lodestar/config"; +import {getNodeLogger} from "@lodestar/logger/node"; +import { + EpochTransitionStep, + PubkeyIndexMap, + StateCloneSource, + StateHashTreeRootSource, +} from "@lodestar/state-transition"; +import {LevelDbController} from "@lodestar/db"; +import {RegistryMetricCreator, collectNodeJSMetrics} from "../../metrics/index.js"; +import {JobFnQueue} from "../../util/queue/fnQueue.js"; +import {QueueMetrics} from "../../util/queue/options.js"; +import {BeaconDb} from "../../db/index.js"; +import { + HistoricalStateRegenMetrics, + HistoricalStateWorkerApi, + HistoricalStateWorkerData, + RegenErrorType, +} from "./types.js"; +import {getHistoricalState} from "./getHistoricalState.js"; + +// most of this setup copied from networkCoreWorker.ts + +const workerData = worker.workerData as HistoricalStateWorkerData; + +// TODO: Pass options from main thread for logging +// TODO: Logging won't be visible in file loggers +const logger = getNodeLogger(workerData.loggerOpts); + +logger.info("Historical state worker started"); + +const config = createBeaconConfig(chainConfigFromJson(workerData.chainConfigJson), workerData.genesisValidatorsRoot); + +const db = new BeaconDb(config, await LevelDbController.create({name: workerData.dbLocation}, {logger})); + +const abortController = new AbortController(); + +// Set up metrics, nodejs, state transition, queue +const metricsRegister = workerData.metricsEnabled ? new RegistryMetricCreator() : null; +let historicalStateRegenMetrics: HistoricalStateRegenMetrics | undefined; +let queueMetrics: QueueMetrics | undefined; +if (metricsRegister) { + const closeMetrics = collectNodeJSMetrics(metricsRegister, "lodestar_historical_state_worker_"); + abortController.signal.addEventListener("abort", closeMetrics, {once: true}); + + historicalStateRegenMetrics = { + // state transition metrics + epochTransitionTime: metricsRegister.histogram({ + name: "lodestar_historical_state_stfn_epoch_transition_seconds", + help: "Time to process a single epoch transition in seconds", + // Epoch transitions are 100ms on very fast clients, and average 800ms on heavy networks + buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1, 1.25, 1.5, 3, 10], + }), + epochTransitionCommitTime: metricsRegister.histogram({ + name: "lodestar_historical_state_stfn_epoch_transition_commit_seconds", + help: "Time to call commit after process a single epoch transition in seconds", + buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1], + }), + epochTransitionStepTime: metricsRegister.histogram<{step: EpochTransitionStep}>({ + name: "lodestar_historical_state_stfn_epoch_transition_step_seconds", + help: "Time to call each step of epoch transition in seconds", + labelNames: ["step"], + buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1], + }), + processBlockTime: metricsRegister.histogram({ + name: "lodestar_historical_state_stfn_process_block_seconds", + help: "Time to process a single block in seconds", + // TODO: Add metrics for each step + // Block processing can take 5-40ms, 100ms max + buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1], + }), + processBlockCommitTime: metricsRegister.histogram({ + name: "lodestar_historical_state_stfn_process_block_commit_seconds", + help: "Time to call commit after process a single block in seconds", + buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1], + }), + stateHashTreeRootTime: metricsRegister.histogram<{source: StateHashTreeRootSource}>({ + name: "lodestar_historical_state_stfn_hash_tree_root_seconds", + help: "Time to compute the hash tree root of a post state in seconds", + buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5], + labelNames: ["source"], + }), + preStateBalancesNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({ + name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total", + help: "Total count state.balances nodesPopulated is false on stfn", + labelNames: ["source"], + }), + preStateBalancesNodesPopulatedHit: metricsRegister.gauge<{source: StateCloneSource}>({ + name: "lodestar_historical_state_stfn_balances_nodes_populated_hit_total", + help: "Total count state.balances nodesPopulated is true on stfn", + labelNames: ["source"], + }), + preStateValidatorsNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({ + name: "lodestar_historical_state_stfn_validators_nodes_populated_miss_total", + help: "Total count state.validators nodesPopulated is false on stfn", + labelNames: ["source"], + }), + preStateValidatorsNodesPopulatedHit: metricsRegister.gauge<{source: StateCloneSource}>({ + name: "lodestar_historical_state_stfn_validators_nodes_populated_hit_total", + help: "Total count state.validators nodesPopulated is true on stfn", + labelNames: ["source"], + }), + preStateClonedCount: metricsRegister.histogram({ + name: "lodestar_historical_state_stfn_state_cloned_count", + help: "Histogram of cloned count per state every time state.clone() is called", + buckets: [1, 2, 5, 10, 50, 250], + }), + postStateBalancesNodesPopulatedHit: metricsRegister.gauge({ + name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_hit_total", + help: "Total count state.validators nodesPopulated is true on stfn for post state", + }), + postStateBalancesNodesPopulatedMiss: metricsRegister.gauge({ + name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_miss_total", + help: "Total count state.validators nodesPopulated is false on stfn for post state", + }), + postStateValidatorsNodesPopulatedHit: metricsRegister.gauge({ + name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_hit_total", + help: "Total count state.validators nodesPopulated is true on stfn for post state", + }), + postStateValidatorsNodesPopulatedMiss: metricsRegister.gauge({ + name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_miss_total", + help: "Total count state.validators nodesPopulated is false on stfn for post state", + }), + registerValidatorStatuses: () => {}, + + // historical state regen metrics + regenTime: metricsRegister.histogram({ + name: "lodestar_historical_state_regen_time_seconds", + help: "Time to regenerate a historical state in seconds", + // Historical state regen can take up to 3h as of Aug 2024 + // 5m, 10m, 30m, 1h, 3h + buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60], + }), + loadStateTime: metricsRegister.histogram({ + name: "lodestar_historical_state_load_nearest_state_time_seconds", + help: "Time to load a nearest historical state from the database in seconds", + // 30s, 1m, 2m, 4m + buckets: [30, 60, 120, 240], + }), + stateTransitionTime: metricsRegister.histogram({ + name: "lodestar_historical_state_state_transition_time_seconds", + help: "Time to run state transition to regen historical state in seconds", + // 5m, 10m, 30m, 1h, 3h + buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60], + }), + stateTransitionBlocks: metricsRegister.histogram({ + name: "lodestar_historical_state_state_transition_blocks", + help: "Count of blocks processed during state transition to regen historical state", + // given archiveStateEpochFrequency=1024, it could process up to 32768 blocks + buckets: [10, 100, 1000, 10000, 30000], + }), + stateSerializationTime: metricsRegister.histogram({ + name: "lodestar_historical_state_serialization_time_seconds", + help: "Time to serialize a historical state in seconds", + buckets: [0.25, 0.5, 1, 2], + }), + regenRequestCount: metricsRegister.gauge({ + name: "lodestar_historical_state_request_count", + help: "Count of total historical state requests", + }), + regenSuccessCount: metricsRegister.gauge({ + name: "lodestar_historical_state_success_count", + help: "Count of successful historical state regen", + }), + regenErrorCount: metricsRegister.gauge<{reason: RegenErrorType}>({ + name: "lodestar_historical_state_error_count", + help: "Count of failed historical state regen", + labelNames: ["reason"], + }), + }; + + queueMetrics = { + length: metricsRegister.gauge({ + name: "lodestar_historical_state_queue_length", + help: "Count of total regen queue length", + }), + droppedJobs: metricsRegister.gauge({ + name: "lodestar_historical_state_queue_dropped_jobs_total", + help: "Count of total regen queue dropped jobs", + }), + jobTime: metricsRegister.histogram({ + name: "lodestar_historical_state_queue_job_time_seconds", + help: "Time to process regen queue job in seconds", + buckets: [0.01, 0.1, 1, 10, 100], + }), + jobWaitTime: metricsRegister.histogram({ + name: "lodestar_historical_state_queue_job_wait_time_seconds", + help: "Time from job added to the regen queue to starting in seconds", + buckets: [0.01, 0.1, 1, 10, 100], + }), + concurrency: metricsRegister.gauge({ + name: "lodestar_historical_state_queue_concurrency", + help: "Current concurrency of regen queue", + }), + }; +} + +const queue = new JobFnQueue( + { + maxConcurrency: workerData.maxConcurrency, + maxLength: workerData.maxLength, + signal: abortController.signal, + }, + queueMetrics +); + +const pubkey2index = new PubkeyIndexMap(); + +const api: HistoricalStateWorkerApi = { + async close() { + abortController.abort(); + }, + async scrapeMetrics() { + return metricsRegister?.metrics() ?? ""; + }, + async getHistoricalState(slot) { + historicalStateRegenMetrics?.regenRequestCount.inc(); + + const stateBytes = await queue.push(() => + getHistoricalState(slot, config, db, pubkey2index, historicalStateRegenMetrics) + ); + const result = Transfer(stateBytes, [stateBytes.buffer]) as unknown as Uint8Array; + + historicalStateRegenMetrics?.regenSuccessCount.inc(); + return result; + }, +}; + +expose(api); diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index aa70b0bdcce4..3bde2e4acb7a 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -142,6 +142,10 @@ export interface IBeaconChain { getHeadStateAtCurrentEpoch(regenCaller: RegenCaller): Promise; getHeadStateAtEpoch(epoch: Epoch, regenCaller: RegenCaller): Promise; + getHistoricalStateBySlot( + slot: Slot + ): Promise<{state: Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>; + /** Returns a local state canonical at `slot` */ getStateBySlot( slot: Slot, diff --git a/packages/beacon-node/src/metrics/server/http.ts b/packages/beacon-node/src/metrics/server/http.ts index d8fbb289e951..e197dcfa2a25 100644 --- a/packages/beacon-node/src/metrics/server/http.ts +++ b/packages/beacon-node/src/metrics/server/http.ts @@ -24,9 +24,9 @@ export async function getHttpMetricsServer( opts: HttpMetricsServerOpts, { register, - getOtherMetrics = async () => "", + getOtherMetrics = async () => [], logger, - }: {register: Registry; getOtherMetrics?: () => Promise; logger: Logger} + }: {register: Registry; getOtherMetrics?: () => Promise; logger: Logger} ): Promise { // New registry to metric the metrics. Using the same registry would deadlock the .metrics promise const httpServerRegister = new RegistryMetricCreator(); @@ -53,7 +53,8 @@ export async function getHttpMetricsServer( } else { // Get scrape time metrics const httpServerMetrics = await httpServerRegister.metrics(); - const metricsStr = `${metricsRes[0].result}\n\n${metricsRes[1]}\n\n${httpServerMetrics}`; + const metrics = [metricsRes[0].result, httpServerMetrics, ...metricsRes[1]]; + const metricsStr = metrics.join("\n\n"); res.writeHead(200, {"content-type": register.contentType}).end(metricsStr); } } else { diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index a1147b60bea2..088541a6b5d5 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -21,6 +21,7 @@ import {getApi, BeaconRestApiServer} from "../api/index.js"; import {initializeExecutionEngine, initializeExecutionBuilder} from "../execution/index.js"; import {initializeEth1ForBlockProduction} from "../eth1/index.js"; import {initCKZG, loadEthereumTrustedSetup, TrustedFileMode} from "../util/kzg.js"; +import {HistoricalStateRegen} from "../chain/historicalState/index.js"; import {IBeaconNodeOptions} from "./options.js"; import {runNodeNotifier} from "./notifier.js"; @@ -194,6 +195,17 @@ export class BeaconNode { ) : null; + const historicalStateRegen = await HistoricalStateRegen.init({ + opts: { + genesisTime: anchorState.genesisTime, + dbLocation: opts.db.name, + }, + config, + metrics, + logger: logger.child({module: LoggerModule.chain}), + signal, + }); + const chain = new BeaconChain(opts.chain, { config, db, @@ -216,6 +228,7 @@ export class BeaconNode { executionBuilder: opts.executionBuilder.enabled ? initializeExecutionBuilder(opts.executionBuilder, config, metrics, logger) : undefined, + historicalStateRegen, }); // Load persisted data from disk to in-memory caches @@ -274,7 +287,7 @@ export class BeaconNode { const metricsServer = opts.metrics.enabled ? await getHttpMetricsServer(opts.metrics, { register: (metrics as Metrics).register, - getOtherMetrics: () => network.scrapeMetrics(), + getOtherMetrics: async () => Promise.all([network.scrapeMetrics(), historicalStateRegen.scrapeMetrics()]), logger: logger.child({module: LoggerModule.metrics}), }) : null; diff --git a/packages/db/src/controller/level.ts b/packages/db/src/controller/level.ts index 07ebef8e8a22..084b577b6a01 100644 --- a/packages/db/src/controller/level.ts +++ b/packages/db/src/controller/level.ts @@ -44,7 +44,12 @@ export class LevelDbController implements DatabaseController { const db = - opts.db || new ClassicLevel(opts.name || "beaconchain", {keyEncoding: "binary", valueEncoding: "binary"}); + opts.db || + new ClassicLevel(opts.name || "beaconchain", { + keyEncoding: "binary", + valueEncoding: "binary", + multithreading: true, + }); try { await db.open(); From 4c3199a42eb9b88a38f660f550ea4eea905eb1a7 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 30 Jul 2024 13:45:19 +0100 Subject: [PATCH 2/3] fix: sanitize url in http client debug logs and metrics (#6974) * fix: sanitize url in http client debug logs and metrics * Improve naming consistency * Clean up * Re-assign baseUrl to fix type inference --- packages/api/src/utils/client/httpClient.ts | 31 ++++++++++++------- packages/api/src/utils/client/request.ts | 7 ++++- .../src/eth1/provider/eth1Provider.ts | 4 +-- .../beacon-node/src/execution/builder/http.ts | 4 +-- .../beacon-node/src/execution/engine/index.ts | 4 +-- .../src/cmds/validator/signers/logSigners.ts | 4 +-- packages/utils/src/url.ts | 4 +-- .../src/services/externalSignerSync.ts | 4 +-- packages/validator/src/validator.ts | 6 ++-- 9 files changed, 41 insertions(+), 27 deletions(-) diff --git a/packages/api/src/utils/client/httpClient.ts b/packages/api/src/utils/client/httpClient.ts index 27757aa0846c..e11e35dc85c7 100644 --- a/packages/api/src/utils/client/httpClient.ts +++ b/packages/api/src/utils/client/httpClient.ts @@ -1,4 +1,4 @@ -import {ErrorAborted, Logger, MapDef, TimeoutError, isValidHttpUrl, retry} from "@lodestar/utils"; +import {ErrorAborted, Logger, MapDef, TimeoutError, isValidHttpUrl, retry, toPrintableUrl} from "@lodestar/utils"; import {mergeHeaders} from "../headers.js"; import {Endpoint} from "../types.js"; import {WireFormat} from "../wireFormat.js"; @@ -115,7 +115,12 @@ export class HttpClient implements IHttpClient { } // De-duplicate by baseUrl, having two baseUrls with different token or timeouts does not make sense if (!this.urlsInits.some((opt) => opt.baseUrl === urlInit.baseUrl)) { - this.urlsInits.push({...urlInit, urlIndex: i} as UrlInitRequired); + this.urlsInits.push({ + ...urlInit, + baseUrl: urlInit.baseUrl, + urlIndex: i, + printableUrl: toPrintableUrl(urlInit.baseUrl), + }); } } @@ -134,7 +139,7 @@ export class HttpClient implements IHttpClient { if (metrics) { metrics.urlsScore.addCollect(() => { for (let i = 0; i < this.urlsScore.length; i++) { - metrics.urlsScore.set({urlIndex: i, baseUrl: this.urlsInits[i].baseUrl}, this.urlsScore[i]); + metrics.urlsScore.set({urlIndex: i, baseUrl: this.urlsInits[i].printableUrl}, this.urlsScore[i]); } }); } @@ -185,12 +190,12 @@ export class HttpClient implements IHttpClient { // - If url[0] is good, only send to 0 // - If url[0] has recently errored, send to both 0, 1, etc until url[0] does not error for some time for (; i < this.urlsInits.length; i++) { - const baseUrl = this.urlsInits[i].baseUrl; + const {printableUrl} = this.urlsInits[i]; const routeId = definition.operationId; if (i > 0) { - this.metrics?.requestToFallbacks.inc({routeId, baseUrl}); - this.logger?.debug("Requesting fallback URL", {routeId, baseUrl, score: this.urlsScore[i]}); + this.metrics?.requestToFallbacks.inc({routeId, baseUrl: printableUrl}); + this.logger?.debug("Requesting fallback URL", {routeId, baseUrl: printableUrl, score: this.urlsScore[i]}); } // eslint-disable-next-line @typescript-eslint/naming-convention @@ -217,7 +222,11 @@ export class HttpClient implements IHttpClient { if (++errorCount >= requestCount) { resolve(res); } else { - this.logger?.debug("Request error, retrying", {routeId, baseUrl}, res.error() as Error); + this.logger?.debug( + "Request error, retrying", + {routeId, baseUrl: printableUrl}, + res.error() as Error + ); } } }, @@ -229,7 +238,7 @@ export class HttpClient implements IHttpClient { if (++errorCount >= requestCount) { reject(err); } else { - this.logger?.debug("Request error, retrying", {routeId, baseUrl}, err); + this.logger?.debug("Request error, retrying", {routeId, baseUrl: printableUrl}, err); } } ); @@ -347,7 +356,7 @@ export class HttpClient implements IHttpClient { abortSignals.forEach((s) => s?.addEventListener("abort", onSignalAbort)); const routeId = definition.operationId; - const {baseUrl, requestWireFormat, responseWireFormat} = init; + const {printableUrl, requestWireFormat, responseWireFormat} = init; const timer = this.metrics?.requestTime.startTimer({routeId}); try { @@ -359,7 +368,7 @@ export class HttpClient implements IHttpClient { if (!apiResponse.ok) { await apiResponse.errorBody(); this.logger?.debug("API response error", {routeId, status: apiResponse.status}); - this.metrics?.requestErrors.inc({routeId, baseUrl}); + this.metrics?.requestErrors.inc({routeId, baseUrl: printableUrl}); return apiResponse; } @@ -376,7 +385,7 @@ export class HttpClient implements IHttpClient { streamTimer?.(); } } catch (e) { - this.metrics?.requestErrors.inc({routeId, baseUrl}); + this.metrics?.requestErrors.inc({routeId, baseUrl: printableUrl}); if (isAbortedError(e)) { if (abortSignals.some((s) => s?.aborted)) { diff --git a/packages/api/src/utils/client/request.ts b/packages/api/src/utils/client/request.ts index 9c36ad111972..2040f1bbc5a9 100644 --- a/packages/api/src/utils/client/request.ts +++ b/packages/api/src/utils/client/request.ts @@ -28,7 +28,12 @@ export type OptionalRequestInit = { }; export type UrlInit = ApiRequestInit & {baseUrl?: string}; -export type UrlInitRequired = ApiRequestInit & {urlIndex: number; baseUrl: string}; +export type UrlInitRequired = ApiRequestInit & { + urlIndex: number; + baseUrl: string; + /** Used in logs and metrics to prevent leaking user credentials */ + printableUrl: string; +}; export type ApiRequestInit = ExtraRequestInit & OptionalRequestInit & RequestInit; export type ApiRequestInitRequired = Required & UrlInitRequired; diff --git a/packages/beacon-node/src/eth1/provider/eth1Provider.ts b/packages/beacon-node/src/eth1/provider/eth1Provider.ts index 493884c0b5f0..3af909cd132e 100644 --- a/packages/beacon-node/src/eth1/provider/eth1Provider.ts +++ b/packages/beacon-node/src/eth1/provider/eth1Provider.ts @@ -1,7 +1,7 @@ import {toHexString} from "@chainsafe/ssz"; import {phase0} from "@lodestar/types"; import {ChainConfig} from "@lodestar/config"; -import {fromHex, isErrorAborted, createElapsedTimeTracker, toSafePrintableUrl} from "@lodestar/utils"; +import {fromHex, isErrorAborted, createElapsedTimeTracker, toPrintableUrl} from "@lodestar/utils"; import {Logger} from "@lodestar/logger"; import {FetchError, isFetchError} from "@lodestar/api"; @@ -84,7 +84,7 @@ export class Eth1Provider implements IEth1Provider { jwtVersion: opts.jwtVersion, metrics: metrics, }); - this.logger?.info("Eth1 provider", {urls: providerUrls.map(toSafePrintableUrl).toString()}); + this.logger?.info("Eth1 provider", {urls: providerUrls.map(toPrintableUrl).toString()}); this.rpc.emitter.on(JsonRpcHttpClientEvent.RESPONSE, () => { const oldState = this.state; diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index 9791f93a8ee2..934b874f5ae3 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -14,7 +14,7 @@ import {ChainForkConfig} from "@lodestar/config"; import {Logger} from "@lodestar/logger"; import {getClient, ApiClient as BuilderApi} from "@lodestar/api/builder"; import {SLOTS_PER_EPOCH, ForkExecution} from "@lodestar/params"; -import {toSafePrintableUrl} from "@lodestar/utils"; +import {toPrintableUrl} from "@lodestar/utils"; import {Metrics} from "../../metrics/metrics.js"; import {IExecutionBuilder} from "./interface.js"; @@ -64,7 +64,7 @@ export class ExecutionBuilderHttp implements IExecutionBuilder { }, {config, metrics: metrics?.builderHttpClient} ); - logger?.info("External builder", {url: toSafePrintableUrl(baseUrl)}); + logger?.info("External builder", {url: toPrintableUrl(baseUrl)}); this.config = config; this.issueLocalFcUWithFeeRecipient = opts.issueLocalFcUWithFeeRecipient; diff --git a/packages/beacon-node/src/execution/engine/index.ts b/packages/beacon-node/src/execution/engine/index.ts index 2d92a439c86d..581933d24296 100644 --- a/packages/beacon-node/src/execution/engine/index.ts +++ b/packages/beacon-node/src/execution/engine/index.ts @@ -1,4 +1,4 @@ -import {fromHex, toSafePrintableUrl} from "@lodestar/utils"; +import {fromHex, toPrintableUrl} from "@lodestar/utils"; import {JsonRpcHttpClient} from "../../eth1/provider/jsonRpcHttpClient.js"; import {IExecutionEngine} from "./interface.js"; import {ExecutionEngineDisabled} from "./disabled.js"; @@ -39,7 +39,7 @@ export function getExecutionEngineHttp( jwtId: opts.jwtId, jwtVersion: opts.jwtVersion, }); - modules.logger.info("Execution client", {urls: opts.urls.map(toSafePrintableUrl).toString()}); + modules.logger.info("Execution client", {urls: opts.urls.map(toPrintableUrl).toString()}); return new ExecutionEngineHttp(rpc, modules); } diff --git a/packages/cli/src/cmds/validator/signers/logSigners.ts b/packages/cli/src/cmds/validator/signers/logSigners.ts index 47b0edc3a4d8..20aa50d25ff6 100644 --- a/packages/cli/src/cmds/validator/signers/logSigners.ts +++ b/packages/cli/src/cmds/validator/signers/logSigners.ts @@ -1,5 +1,5 @@ import {Signer, SignerLocal, SignerRemote, SignerType} from "@lodestar/validator"; -import {LogLevel, Logger, toSafePrintableUrl} from "@lodestar/utils"; +import {LogLevel, Logger, toPrintableUrl} from "@lodestar/utils"; import {YargsError} from "../../../util/errors.js"; import {IValidatorCliArgs} from "../options.js"; @@ -29,7 +29,7 @@ export function logSigners(logger: Pick, signers: Signer[ } for (const {url, pubkeys} of groupRemoteSignersByUrl(remoteSigners)) { - logger.info(`Remote signers on URL: ${toSafePrintableUrl(url)}`); + logger.info(`Remote signers on URL: ${toPrintableUrl(url)}`); for (const pubkey of pubkeys) { logger.info(pubkey); } diff --git a/packages/utils/src/url.ts b/packages/utils/src/url.ts index 7d0b23347617..f00b70edbde2 100644 --- a/packages/utils/src/url.ts +++ b/packages/utils/src/url.ts @@ -20,10 +20,10 @@ export function isValidHttpUrl(urlStr: string): boolean { } /** - * Sanitize URL to prevent leaking user credentials in logs + * Sanitize URL to prevent leaking user credentials in logs or metrics * * Note: `urlStr` must be a valid URL */ -export function toSafePrintableUrl(urlStr: string): string { +export function toPrintableUrl(urlStr: string): string { return new URL(urlStr).origin; } diff --git a/packages/validator/src/services/externalSignerSync.ts b/packages/validator/src/services/externalSignerSync.ts index 03db612249b4..2f1880dda1bb 100644 --- a/packages/validator/src/services/externalSignerSync.ts +++ b/packages/validator/src/services/externalSignerSync.ts @@ -2,7 +2,7 @@ import {fromHexString} from "@chainsafe/ssz"; import {PublicKey} from "@chainsafe/blst"; import {ChainForkConfig} from "@lodestar/config"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {toSafePrintableUrl} from "@lodestar/utils"; +import {toPrintableUrl} from "@lodestar/utils"; import {LoggerVc} from "../util/index.js"; import {externalSignerGetKeys} from "../util/externalSignerClient.js"; @@ -35,7 +35,7 @@ export function pollExternalSignerPubkeys( async function fetchExternalSignerPubkeys(): Promise { // External signer URL is already validated earlier const externalSignerUrl = externalSigner.url as string; - const printableUrl = toSafePrintableUrl(externalSignerUrl); + const printableUrl = toPrintableUrl(externalSignerUrl); try { logger.debug("Fetching public keys from external signer", {url: printableUrl}); diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 785d3de1db2a..461762560fee 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -2,7 +2,7 @@ import {toHexString} from "@chainsafe/ssz"; import {BLSPubkey, phase0, ssz} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {Genesis} from "@lodestar/types/phase0"; -import {Logger, toSafePrintableUrl} from "@lodestar/utils"; +import {Logger, toPrintableUrl} from "@lodestar/utils"; import {getClient, ApiClient, routes, ApiRequestInit, defaultInit} from "@lodestar/api"; import {computeEpochAtSlot, getCurrentSlot} from "@lodestar/state-transition"; import {Clock, IClock} from "./util/clock.js"; @@ -132,7 +132,7 @@ export class Validator { // not be any errors in the logs due to fallback nodes handling the requests const {httpClient} = this.api; if (httpClient.urlsInits.length > 1) { - const primaryNodeUrl = toSafePrintableUrl(httpClient.urlsInits[0].baseUrl); + const primaryNodeUrl = toPrintableUrl(httpClient.urlsInits[0].baseUrl); this.clock.runEveryEpoch(async () => { // Only emit warning if URL score is 0 to prevent false positives @@ -293,7 +293,7 @@ export class Validator { // not necessary since this instance won't be used for validator duties api = getClient({urls, globalInit: {signal: opts.abortController.signal, ...globalInit}}, {config, logger}); logger.info("Beacon node", { - urls: urls.map(toSafePrintableUrl).toString(), + urls: urls.map(toPrintableUrl).toString(), requestWireFormat: globalInit?.requestWireFormat ?? defaultInit.requestWireFormat, responseWireFormat: globalInit?.responseWireFormat ?? defaultInit.responseWireFormat, }); From 48a5a94db54f3ccdb4de167e2fba809a73a8e923 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 30 Jul 2024 14:46:28 +0100 Subject: [PATCH 3/3] docs: add note to local testnet setup (#6984) * docs: add note to local testnet setup * Add Kurtosis to wordlist --- .wordlist.txt | 1 + .../contribution/advanced-topics/setting-up-a-testnet.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.wordlist.txt b/.wordlist.txt index 46ca81441348..7ba20c86be99 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -54,6 +54,7 @@ JSObjects JWT KDE Kubernetes +Kurtosis LGPL LGPLv LMD diff --git a/docs/pages/contribution/advanced-topics/setting-up-a-testnet.md b/docs/pages/contribution/advanced-topics/setting-up-a-testnet.md index f7c4bfece237..04cac75c4b0a 100644 --- a/docs/pages/contribution/advanced-topics/setting-up-a-testnet.md +++ b/docs/pages/contribution/advanced-topics/setting-up-a-testnet.md @@ -6,6 +6,10 @@ title: Setting Up a Testnet To quickly test and run Lodestar we recommend starting a local testnet. We recommend a simple configuration of two beacon nodes with multiple validators. The [dev scripts](https://github.com/ChainSafe/lodestar/tree/unstable/scripts/dev) can used for simplicity but below instructions provide more insights on how it works and include details about different configurations. +:::note +The testnet set up in this guide is meant to be short-lived / ephemeral and should primarily be used for development and testing. Please refer to [Ethereum In a Box](https://github.com/rocknet/ethiab) or [Kurtosis ethereum package](https://github.com/ethpandaops/ethereum-package) to set up a long-lived private network or devnet. +::: + **Terminal 1** Run a beacon node as a **bootnode**, with 8 validators with the following command.