diff --git a/cmd/explorer/main.go b/cmd/explorer/main.go index 5395d88d..2a4c7482 100644 --- a/cmd/explorer/main.go +++ b/cmd/explorer/main.go @@ -72,6 +72,7 @@ func startFrontend() { router.HandleFunc("/epochs", handlers.Epochs).Methods("GET") router.HandleFunc("/epoch/{epoch}", handlers.Epoch).Methods("GET") router.HandleFunc("/slots", handlers.Slots).Methods("GET") + router.HandleFunc("/slots/filtered", handlers.SlotsFiltered).Methods("GET") router.HandleFunc("/slot/{slotOrHash}", handlers.Slot).Methods("GET") router.HandleFunc("/slot/{hash}/blob/{blobIdx}", handlers.SlotBlob).Methods("GET") router.HandleFunc("/search", handlers.Search).Methods("GET") diff --git a/db/db.go b/db/db.go index bde216f0..7ced59f0 100644 --- a/db/db.go +++ b/db/db.go @@ -167,17 +167,15 @@ func ApplyEmbeddedDbSchema(version int64) error { goose.SetBaseFS(EmbedPgsqlSchema) engineDialect = "postgres" schemaDirectory = "schema/pgsql" - break case dbtypes.DBEngineSqlite: goose.SetBaseFS(EmbedSqliteSchema) engineDialect = "sqlite3" schemaDirectory = "schema/sqlite" - break default: logger.Fatalf("unknown database engine") } - fmt.Printf(engineDialect) + fmt.Print(engineDialect) if err := goose.SetDialect(engineDialect); err != nil { return err } @@ -240,6 +238,65 @@ func SetExplorerState(key string, value interface{}, tx *sqlx.Tx) error { return nil } +func GetValidatorNames(minIdx uint64, maxIdx uint64, tx *sqlx.Tx) []*dbtypes.ValidatorName { + names := []*dbtypes.ValidatorName{} + err := ReaderDb.Select(&names, `SELECT "index", "name" FROM validator_names WHERE "index" >= $1 AND "index" <= $2`, minIdx, maxIdx) + if err != nil { + logger.Errorf("Error while fetching validator names: %v", err) + return nil + } + return names +} + +func InsertValidatorNames(validatorNames []*dbtypes.ValidatorName, tx *sqlx.Tx) error { + var sql strings.Builder + fmt.Fprint(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: `INSERT INTO validator_names ("index", "name") VALUES `, + dbtypes.DBEngineSqlite: `INSERT OR REPLACE INTO validator_names ("index", "name") VALUES `, + })) + argIdx := 0 + args := make([]any, len(validatorNames)*2) + for i, validatorName := range validatorNames { + if i > 0 { + fmt.Fprintf(&sql, ", ") + } + fmt.Fprintf(&sql, "($%v, $%v)", argIdx+1, argIdx+2) + args[argIdx] = validatorName.Index + args[argIdx+1] = validatorName.Name + argIdx += 2 + } + fmt.Fprint(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: ` ON CONFLICT ("index") DO UPDATE SET name = excluded.name`, + dbtypes.DBEngineSqlite: "", + })) + _, err := tx.Exec(sql.String(), args...) + if err != nil { + return err + } + return nil +} + +func DeleteValidatorNames(validatorNames []uint64, tx *sqlx.Tx) error { + var sql strings.Builder + fmt.Fprint(&sql, `DELETE FROM validator_names WHERE "index" IN (`) + argIdx := 0 + args := make([]any, len(validatorNames)*2) + for i, validatorName := range validatorNames { + if i > 0 { + fmt.Fprintf(&sql, ", ") + } + fmt.Fprintf(&sql, "$%v", argIdx+1) + args[argIdx] = validatorName + argIdx += 1 + } + fmt.Fprint(&sql, ")") + _, err := tx.Exec(sql.String(), args...) + if err != nil { + return err + } + return nil +} + func IsEpochSynchronized(epoch uint64) bool { var count uint64 err := ReaderDb.Get(&count, `SELECT COUNT(*) FROM epochs WHERE epoch = $1`, epoch) @@ -251,7 +308,7 @@ func IsEpochSynchronized(epoch uint64) bool { func InsertSlotAssignments(slotAssignments []*dbtypes.SlotAssignment, tx *sqlx.Tx) error { var sql strings.Builder - fmt.Fprintf(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ + fmt.Fprint(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: "INSERT INTO slot_assignments (slot, proposer) VALUES ", dbtypes.DBEngineSqlite: "INSERT OR REPLACE INTO slot_assignments (slot, proposer) VALUES ", })) @@ -266,7 +323,7 @@ func InsertSlotAssignments(slotAssignments []*dbtypes.SlotAssignment, tx *sqlx.T args[argIdx+1] = slotAssignment.Proposer argIdx += 2 } - fmt.Fprintf(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ + fmt.Fprint(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: " ON CONFLICT (slot) DO UPDATE SET proposer = excluded.proposer", dbtypes.DBEngineSqlite: "", })) @@ -460,45 +517,8 @@ func GetBlocksByParentRoot(parentRoot []byte) []*dbtypes.Block { return blocks } -func GetBlocksWithGraffiti(graffiti string, firstSlot uint64, offset uint64, limit uint32, withOrphaned bool) []*dbtypes.Block { - blocks := []*dbtypes.Block{} - orphanedLimit := "" - if !withOrphaned { - orphanedLimit = "AND NOT orphaned" - } - err := ReaderDb.Select(&blocks, EngineQuery(map[dbtypes.DBEngineType]string{ - dbtypes.DBEnginePgsql: ` - SELECT - root, slot, parent_root, state_root, orphaned, proposer, graffiti, graffiti_text, - attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation - FROM blocks - WHERE graffiti_text ilike $1 AND slot < $2 ` + orphanedLimit + ` - ORDER BY slot DESC - LIMIT $3 OFFSET $4`, - dbtypes.DBEngineSqlite: ` - SELECT - root, slot, parent_root, state_root, orphaned, proposer, graffiti, graffiti_text, - attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation - FROM blocks - WHERE graffiti_text LIKE $1 AND slot < $2 ` + orphanedLimit + ` - ORDER BY slot DESC - LIMIT $3 OFFSET $4`, - }), "%"+graffiti+"%", firstSlot, limit, offset) - if err != nil { - logger.Errorf("Error while fetching blocks with graffiti: %v", err) - return nil - } - return blocks -} - -func GetAssignedBlocks(proposer uint64, firstSlot uint64, offset uint64, limit uint32, withOrphaned bool) []*dbtypes.AssignedBlock { +func GetFilteredBlocks(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint64, limit uint32) []*dbtypes.AssignedBlock { blockAssignments := []*dbtypes.AssignedBlock{} - orphanedLimit := "" - if !withOrphaned { - orphanedLimit = "AND NOT orphaned" - } var sql strings.Builder fmt.Fprintf(&sql, `SELECT slot_assignments.slot, slot_assignments.proposer`) blockFields := []string{ @@ -509,16 +529,65 @@ func GetAssignedBlocks(proposer uint64, firstSlot uint64, offset uint64, limit u for _, blockField := range blockFields { fmt.Fprintf(&sql, ", blocks.%v AS \"block.%v\"", blockField, blockField) } - fmt.Fprintf(&sql, ` - FROM slot_assignments - LEFT JOIN blocks ON blocks.slot = slot_assignments.slot - WHERE (slot_assignments.proposer = $1 OR blocks.proposer = $1) AND slot_assignments.slot < $2 `+orphanedLimit+` - ORDER BY slot_assignments.slot DESC - LIMIT $3 OFFSET $4 - `) - rows, err := ReaderDb.Query(sql.String(), proposer, firstSlot, limit, offset) - if err != nil { - logger.Errorf("Error while fetching assigned blocks: %v", err) + fmt.Fprintf(&sql, ` FROM slot_assignments `) + fmt.Fprintf(&sql, ` LEFT JOIN blocks ON blocks.slot = slot_assignments.slot `) + if filter.ProposerName != "" { + fmt.Fprintf(&sql, ` LEFT JOIN validator_names ON validator_names."index" = COALESCE(blocks.proposer, slot_assignments.proposer) `) + } + + argIdx := 0 + args := make([]any, 0) + + argIdx++ + fmt.Fprintf(&sql, ` WHERE slot_assignments.slot < $%v `, argIdx) + args = append(args, firstSlot) + + if filter.WithMissing == 0 { + fmt.Fprintf(&sql, ` AND blocks.root IS NOT NULL `) + } else if filter.WithMissing == 2 { + fmt.Fprintf(&sql, ` AND blocks.root IS NULL `) + } + if filter.WithOrphaned == 0 { + fmt.Fprintf(&sql, ` AND (`) + if filter.WithMissing != 0 { + fmt.Fprintf(&sql, `blocks.orphaned IS NULL OR`) + } + fmt.Fprintf(&sql, ` NOT blocks.orphaned) `) + } else if filter.WithOrphaned == 2 { + fmt.Fprintf(&sql, ` AND blocks.orphaned `) + } + if filter.ProposerIndex != nil { + argIdx++ + fmt.Fprintf(&sql, ` AND (slot_assignments.proposer = $%v OR blocks.proposer = $%v) `, argIdx, argIdx) + args = append(args, *filter.ProposerIndex) + } + if filter.Graffiti != "" { + argIdx++ + fmt.Fprintf(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: ` AND blocks.graffiti_text ilike $%v `, + dbtypes.DBEngineSqlite: ` AND blocks.graffiti_text LIKE $%v `, + }), argIdx) + args = append(args, "%"+filter.Graffiti+"%") + } + if filter.ProposerName != "" { + argIdx++ + fmt.Fprintf(&sql, EngineQuery(map[dbtypes.DBEngineType]string{ + dbtypes.DBEnginePgsql: ` AND validator_names.name ilike $%v `, + dbtypes.DBEngineSqlite: ` AND validator_names.name LIKE $%v `, + }), argIdx) + args = append(args, "%"+filter.ProposerName+"%") + } + + fmt.Fprintf(&sql, ` ORDER BY slot_assignments.slot DESC `) + fmt.Fprintf(&sql, ` LIMIT $%v OFFSET $%v `, argIdx+1, argIdx+2) + argIdx += 2 + args = append(args, limit) + args = append(args, offset) + + //fmt.Printf("sql: %v, args: %v\n", sql.String(), args) + rows, err := ReaderDb.Query(sql.String(), args...) + if err != nil { + logger.WithError(err).Errorf("Error while fetching filtered blocks: %v", sql.String()) return nil } diff --git a/db/schema/pgsql/20230902172124_validator-names.sql b/db/schema/pgsql/20230902172124_validator-names.sql new file mode 100644 index 00000000..45a10d21 --- /dev/null +++ b/db/schema/pgsql/20230902172124_validator-names.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE TABLE IF NOT EXISTS public."validator_names" +( + "index" bigint NOT NULL, + "name" character varying(250) NOT NULL, + PRIMARY KEY ("index") +); + +CREATE INDEX IF NOT EXISTS "validator_names_name_idx" + ON public."validator_names" USING gin + ("name" gin_trgm_ops); + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd diff --git a/db/schema/sqlite/20230902172124_validator-names.sql b/db/schema/sqlite/20230902172124_validator-names.sql new file mode 100644 index 00000000..83f2e75f --- /dev/null +++ b/db/schema/sqlite/20230902172124_validator-names.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE TABLE IF NOT EXISTS "validator_names" +( + "index" bigint NOT NULL, + "name" character varying(250) NOT NULL, + PRIMARY KEY ("index") +); + +CREATE INDEX IF NOT EXISTS "validator_names_name_idx" + ON "validator_names" + ("name" ASC); + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index 7d1a967c..3ab7107a 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -5,12 +5,17 @@ type ExplorerState struct { Value string `db:"value"` } +type ValidatorName struct { + Index uint64 `db:"index"` + Name string `db:"name"` +} + type Block struct { Root []byte `db:"root"` Slot uint64 `db:"slot"` ParentRoot []byte `db:"parent_root"` StateRoot []byte `db:"state_root"` - Orphaned bool `db:"orphaned"` + Orphaned uint8 `db:"orphaned"` Proposer uint64 `db:"proposer"` Graffiti []byte `db:"graffiti"` GraffitiText string `db:"graffiti_text"` diff --git a/dbtypes/other.go b/dbtypes/other.go index 277cd1c1..fbcb5b5c 100644 --- a/dbtypes/other.go +++ b/dbtypes/other.go @@ -5,3 +5,11 @@ type AssignedBlock struct { Proposer uint64 `db:"proposer"` Block *Block `db:"block"` } + +type BlockFilter struct { + Graffiti string + ProposerIndex *uint64 + ProposerName string + WithOrphaned uint8 + WithMissing uint8 +} diff --git a/handlers/epoch.go b/handlers/epoch.go index 405d93f1..8f11e6e8 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -132,7 +132,7 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { dbSlot := dbSlots[dbIdx] dbIdx++ blockStatus := uint8(1) - if dbSlot.Orphaned { + if dbSlot.Orphaned == 1 { blockStatus = 2 pageData.OrphanedCount++ } else { diff --git a/handlers/index.go b/handlers/index.go index 631daba7..9d8ca650 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -208,7 +208,7 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, currentSlot continue } blockStatus := 1 - if blockData.Orphaned { + if blockData.Orphaned == 1 { blockStatus = 2 } pageData.RecentBlocks = append(pageData.RecentBlocks, &models.IndexPageDataBlocks{ @@ -253,7 +253,7 @@ func buildIndexPageRecentSlotsData(pageData *models.IndexPageData, firstSlot uin dbSlot := dbSlots[dbIdx] dbIdx++ blockStatus := uint64(1) - if dbSlot.Orphaned { + if dbSlot.Orphaned == 1 { blockStatus = 2 } diff --git a/handlers/search.go b/handlers/search.go index fc9deae5..01a983ac 100644 --- a/handlers/search.go +++ b/handlers/search.go @@ -85,7 +85,7 @@ func Search(w http.ResponseWriter, r *http.Request) { LIMIT 1`, }), "%"+searchQuery+"%") if err == nil { - http.Redirect(w, r, "/slots?q="+searchQuery, http.StatusMovedPermanently) + http.Redirect(w, r, "/filtered?f&f.graffiti="+searchQuery, http.StatusMovedPermanently) return } diff --git a/handlers/slots.go b/handlers/slots.go index 363a200d..3cf7a17c 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -32,25 +32,11 @@ func Slots(w http.ResponseWriter, r *http.Request) { if urlArgs.Has("c") { pageSize, _ = strconv.ParseUint(urlArgs.Get("c"), 10, 64) } - var graffiti string - if urlArgs.Has("q") { - graffiti = urlArgs.Get("q") + var firstSlot uint64 = math.MaxUint64 + if urlArgs.Has("s") { + firstSlot, _ = strconv.ParseUint(urlArgs.Get("s"), 10, 64) } - var pageData *models.SlotsPageData - if graffiti == "" { - var firstSlot uint64 = math.MaxUint64 - if urlArgs.Has("s") { - firstSlot, _ = strconv.ParseUint(urlArgs.Get("s"), 10, 64) - } - pageData = getSlotsPageData(firstSlot, pageSize) - } else { - var pageIdx uint64 = 0 - if urlArgs.Has("s") { - pageIdx, _ = strconv.ParseUint(urlArgs.Get("s"), 10, 64) - } - pageData = getSlotsPageDataWithGraffitiFilter(graffiti, pageIdx, pageSize) - } - data.Data = pageData + data.Data = getSlotsPageData(firstSlot, pageSize) if handleTemplateError(w, r, "slots.go", "Slots", "", pageTemplate.ExecuteTemplate(w, "layout", data)) != nil { return // an error has occurred and was processed @@ -70,9 +56,7 @@ func getSlotsPageData(firstSlot uint64, pageSize uint64) *models.SlotsPageData { func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageData, time.Duration) { logrus.Printf("slots page called: %v:%v", firstSlot, pageSize) - pageData := &models.SlotsPageData{ - ShowForkTree: true, - } + pageData := &models.SlotsPageData{} now := time.Now() currentSlot := utils.TimeToSlot(uint64(now.Unix())) @@ -145,7 +129,7 @@ func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageDat dbSlot := dbSlots[dbIdx] dbIdx++ blockStatus := uint8(1) - if dbSlot.Orphaned { + if dbSlot.Orphaned == 1 { blockStatus = 2 } @@ -334,88 +318,3 @@ func buildSlotsPageSlotGraph(pageData *models.SlotsPageData, slotData *models.Sl } } } - -func getSlotsPageDataWithGraffitiFilter(graffiti string, pageIdx uint64, pageSize uint64) *models.SlotsPageData { - pageData := &models.SlotsPageData{} - pageCacheKey := fmt.Sprintf("slots:%v:%v:g-%v", pageIdx, pageSize, graffiti) - pageData = services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(_ *services.FrontendCacheProcessingPage) interface{} { - return buildSlotsPageDataWithGraffitiFilter(graffiti, pageIdx, pageSize) - }).(*models.SlotsPageData) - return pageData -} - -func buildSlotsPageDataWithGraffitiFilter(graffiti string, pageIdx uint64, pageSize uint64) *models.SlotsPageData { - pageData := &models.SlotsPageData{ - GraffitiFilter: graffiti, - } - logrus.Printf("slots page called (filtered): %v:%v [%v]", pageIdx, pageSize, graffiti) - if pageIdx == 0 { - pageData.IsDefaultPage = true - } - - if pageSize > 100 { - pageSize = 100 - } - pageData.PageSize = pageSize - pageData.TotalPages = pageIdx + 1 - pageData.CurrentPageIndex = pageIdx + 1 - pageData.CurrentPageSlot = pageIdx - if pageIdx >= 1 { - pageData.PrevPageIndex = pageIdx - pageData.PrevPageSlot = pageIdx - 1 - } - pageData.LastPageSlot = 0 - - finalizedEpoch, _ := services.GlobalBeaconService.GetFinalizedEpoch() - - // load slots - pageData.Slots = make([]*models.SlotsPageDataSlot, 0) - dbBlocks := services.GlobalBeaconService.GetDbBlocksByGraffiti(graffiti, pageIdx, uint32(pageSize), true) - haveMore := false - for idx, dbBlock := range dbBlocks { - if idx >= int(pageSize) { - haveMore = true - break - } - slot := dbBlock.Slot - blockStatus := uint8(1) - if dbBlock.Orphaned { - blockStatus = 2 - } - - slotData := &models.SlotsPageDataSlot{ - Slot: slot, - Epoch: utils.EpochOfSlot(slot), - Ts: utils.SlotToTime(slot), - Finalized: finalizedEpoch >= int64(utils.EpochOfSlot(slot)), - Status: blockStatus, - Synchronized: true, - Proposer: dbBlock.Proposer, - ProposerName: services.GlobalBeaconService.GetValidatorName(dbBlock.Proposer), - AttestationCount: dbBlock.AttestationCount, - DepositCount: dbBlock.DepositCount, - ExitCount: dbBlock.ExitCount, - ProposerSlashingCount: dbBlock.ProposerSlashingCount, - AttesterSlashingCount: dbBlock.AttesterSlashingCount, - SyncParticipation: float64(dbBlock.SyncParticipation) * 100, - EthTransactionCount: dbBlock.EthTransactionCount, - EthBlockNumber: dbBlock.EthBlockNumber, - Graffiti: dbBlock.Graffiti, - BlockRoot: dbBlock.Root, - } - pageData.Slots = append(pageData.Slots, slotData) - - } - pageData.SlotCount = uint64(len(pageData.Slots)) - if pageData.SlotCount > 0 { - pageData.FirstSlot = pageData.Slots[0].Slot - pageData.LastSlot = pageData.Slots[pageData.SlotCount-1].Slot - } - if haveMore { - pageData.NextPageIndex = pageIdx + 1 - pageData.NextPageSlot = pageIdx + 1 - pageData.TotalPages++ - } - - return pageData -} diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go new file mode 100644 index 00000000..1ab7e5d1 --- /dev/null +++ b/handlers/slots_filtered.go @@ -0,0 +1,196 @@ +package handlers + +import ( + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/pk910/light-beaconchain-explorer/dbtypes" + "github.com/pk910/light-beaconchain-explorer/services" + "github.com/pk910/light-beaconchain-explorer/templates" + "github.com/pk910/light-beaconchain-explorer/types/models" + "github.com/pk910/light-beaconchain-explorer/utils" + "github.com/sirupsen/logrus" +) + +// SlotsFiltered will return the filtered "slots" page using a go template +func SlotsFiltered(w http.ResponseWriter, r *http.Request) { + var slotsTemplateFiles = append(layoutTemplateFiles, + "slots_filtered/slots_filtered.html", + "_svg/professor.html", + ) + + var pageTemplate = templates.GetTemplate(slotsTemplateFiles...) + + w.Header().Set("Content-Type", "text/html") + data := InitPageData(w, r, "blockchain", "/slots/filtered", "Filtered Slots", slotsTemplateFiles) + + urlArgs := r.URL.Query() + var pageSize uint64 = 50 + if urlArgs.Has("c") { + pageSize, _ = strconv.ParseUint(urlArgs.Get("c"), 10, 64) + } + var pageIdx uint64 = 0 + if urlArgs.Has("s") { + pageIdx, _ = strconv.ParseUint(urlArgs.Get("s"), 10, 64) + } + + var graffiti string + var proposer string + var pname string + var withOrphaned uint64 + var withMissing uint64 + + if urlArgs.Has("f") { + if urlArgs.Has("f.graffiti") { + graffiti = urlArgs.Get("f.graffiti") + } + if urlArgs.Has("f.proposer") { + proposer = urlArgs.Get("f.proposer") + } + if urlArgs.Has("f.pname") { + pname = urlArgs.Get("f.pname") + } + if urlArgs.Has("f.orphaned") { + withOrphaned, _ = strconv.ParseUint(urlArgs.Get("f.orphaned"), 10, 64) + } + if urlArgs.Has("f.missing") { + withMissing, _ = strconv.ParseUint(urlArgs.Get("f.missing"), 10, 64) + } + } else { + withOrphaned = 1 + withMissing = 1 + } + data.Data = getFilteredSlotsPageData(pageIdx, pageSize, graffiti, proposer, pname, uint8(withOrphaned), uint8(withMissing)) + + if handleTemplateError(w, r, "slots_filtered.go", "SlotsFiltered", "", pageTemplate.ExecuteTemplate(w, "layout", data)) != nil { + return // an error has occurred and was processed + } +} + +func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, proposer string, pname string, withOrphaned uint8, withMissing uint8) *models.SlotsFilteredPageData { + pageData := &models.SlotsFilteredPageData{} + pageCacheKey := fmt.Sprintf("slots_filtered:%v:%v:%v:%v:%v:%v:%v", pageIdx, pageSize, graffiti, proposer, pname, withOrphaned, withMissing) + pageData = services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(_ *services.FrontendCacheProcessingPage) interface{} { + return buildFilteredSlotsPageData(pageIdx, pageSize, graffiti, proposer, pname, withOrphaned, withMissing) + }).(*models.SlotsFilteredPageData) + return pageData +} + +func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, proposer string, pname string, withOrphaned uint8, withMissing uint8) *models.SlotsFilteredPageData { + filterArgs := url.Values{} + if graffiti != "" { + filterArgs.Add("f.graffiti", graffiti) + } + if proposer != "" { + filterArgs.Add("f.proposer", proposer) + } + if pname != "" { + filterArgs.Add("f.pname", pname) + } + if withOrphaned != 0 { + filterArgs.Add("f.orphaned", fmt.Sprintf("%v", withOrphaned)) + } + if withMissing != 0 { + filterArgs.Add("f.missing", fmt.Sprintf("%v", withMissing)) + } + + pageData := &models.SlotsFilteredPageData{ + FilterGraffiti: graffiti, + FilterProposer: proposer, + FilterProposerName: pname, + FilterWithOrphaned: withOrphaned, + FilterWithMissing: withMissing, + } + logrus.Printf("slots_filtered page called: %v:%v [%v]", pageIdx, pageSize, graffiti) + if pageIdx == 0 { + pageData.IsDefaultPage = true + } + + if pageSize > 100 { + pageSize = 100 + } + pageData.PageSize = pageSize + pageData.TotalPages = pageIdx + 1 + pageData.CurrentPageIndex = pageIdx + 1 + pageData.CurrentPageSlot = pageIdx + if pageIdx >= 1 { + pageData.PrevPageIndex = pageIdx + pageData.PrevPageSlot = pageIdx - 1 + } + pageData.LastPageSlot = 0 + + finalizedEpoch, _ := services.GlobalBeaconService.GetFinalizedEpoch() + currentSlot := utils.TimeToSlot(uint64(time.Now().Unix())) + + // load slots + pageData.Slots = make([]*models.SlotsFilteredPageDataSlot, 0) + blockFilter := &dbtypes.BlockFilter{ + Graffiti: graffiti, + ProposerName: pname, + WithOrphaned: withOrphaned, + WithMissing: withMissing, + } + if proposer != "" { + pidx, _ := strconv.ParseUint(proposer, 10, 64) + blockFilter.ProposerIndex = &pidx + } + + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(blockFilter, pageIdx, uint32(pageSize)) + haveMore := false + for idx, dbBlock := range dbBlocks { + if idx >= int(pageSize) { + haveMore = true + break + } + slot := dbBlock.Slot + + slotData := &models.SlotsFilteredPageDataSlot{ + Slot: slot, + Epoch: utils.EpochOfSlot(slot), + Ts: utils.SlotToTime(slot), + Finalized: finalizedEpoch >= int64(utils.EpochOfSlot(slot)), + Synchronized: true, + Scheduled: slot >= currentSlot, + Proposer: dbBlock.Proposer, + ProposerName: services.GlobalBeaconService.GetValidatorName(dbBlock.Proposer), + } + + if dbBlock.Block != nil { + slotData.Status = 1 + if dbBlock.Block.Orphaned == 1 { + slotData.Status = 2 + } + slotData.AttestationCount = dbBlock.Block.AttestationCount + slotData.DepositCount = dbBlock.Block.DepositCount + slotData.ExitCount = dbBlock.Block.ExitCount + slotData.ProposerSlashingCount = dbBlock.Block.ProposerSlashingCount + slotData.AttesterSlashingCount = dbBlock.Block.AttesterSlashingCount + slotData.SyncParticipation = float64(dbBlock.Block.SyncParticipation) * 100 + slotData.EthTransactionCount = dbBlock.Block.EthTransactionCount + slotData.EthBlockNumber = dbBlock.Block.EthBlockNumber + slotData.Graffiti = dbBlock.Block.Graffiti + slotData.BlockRoot = dbBlock.Block.Root + } + pageData.Slots = append(pageData.Slots, slotData) + } + pageData.SlotCount = uint64(len(pageData.Slots)) + if pageData.SlotCount > 0 { + pageData.FirstSlot = pageData.Slots[0].Slot + pageData.LastSlot = pageData.Slots[pageData.SlotCount-1].Slot + } + if haveMore { + pageData.NextPageIndex = pageIdx + 1 + pageData.NextPageSlot = pageIdx + 1 + pageData.TotalPages++ + } + + pageData.FirstPageLink = fmt.Sprintf("/slots/filtered?%v&c=%v", filterArgs.Encode(), pageData.PageSize) + pageData.PrevPageLink = fmt.Sprintf("/slots/filtered?%v&c=%v&s=%v", filterArgs.Encode(), pageData.PageSize, pageData.PrevPageSlot) + pageData.NextPageLink = fmt.Sprintf("/slots/filtered?%v&c=%v&s=%v", filterArgs.Encode(), pageData.PageSize, pageData.NextPageSlot) + pageData.LastPageLink = fmt.Sprintf("/slots/filtered?%v&c=%v&s=%v", filterArgs.Encode(), pageData.PageSize, pageData.LastPageSlot) + + return pageData +} diff --git a/handlers/validator.go b/handlers/validator.go index 3a5ba78d..51276965 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -12,6 +12,7 @@ import ( "github.com/gorilla/mux" "github.com/sirupsen/logrus" + "github.com/pk910/light-beaconchain-explorer/dbtypes" "github.com/pk910/light-beaconchain-explorer/rpctypes" "github.com/pk910/light-beaconchain-explorer/services" "github.com/pk910/light-beaconchain-explorer/templates" @@ -149,12 +150,16 @@ func buildValidatorPageData(validatorIndex uint64) (*models.ValidatorPageData, t // load latest blocks pageData.RecentBlocks = make([]*models.ValidatorPageDataBlocks, 0) - blocksData := services.GlobalBeaconService.GetDbBlocksByProposer(validatorIndex, 0, 10, true, true) + blocksData := services.GlobalBeaconService.GetDbBlocksByFilter(&dbtypes.BlockFilter{ + ProposerIndex: &validatorIndex, + WithOrphaned: 1, + WithMissing: 1, + }, 0, 10) for _, blockData := range blocksData { blockStatus := 1 if blockData.Block == nil { blockStatus = 0 - } else if blockData.Block.Orphaned { + } else if blockData.Block.Orphaned == 1 { blockStatus = 2 } blockEntry := models.ValidatorPageDataBlocks{ diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index 50d161d7..d5b24c22 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -9,6 +9,7 @@ import ( "github.com/gorilla/mux" "github.com/sirupsen/logrus" + "github.com/pk910/light-beaconchain-explorer/dbtypes" "github.com/pk910/light-beaconchain-explorer/services" "github.com/pk910/light-beaconchain-explorer/templates" "github.com/pk910/light-beaconchain-explorer/types/models" @@ -88,7 +89,11 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint // load slots pageData.Slots = make([]*models.ValidatorSlotsPageDataSlot, 0) - dbBlocks := services.GlobalBeaconService.GetDbBlocksByProposer(validator, pageIdx, uint32(pageSize), true, true) + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(&dbtypes.BlockFilter{ + ProposerIndex: &validator, + WithOrphaned: 1, + WithMissing: 1, + }, pageIdx, uint32(pageSize)) haveMore := false for idx, blockAssignment := range dbBlocks { if idx >= int(pageSize) { @@ -110,7 +115,7 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint if blockAssignment.Block != nil { dbBlock := blockAssignment.Block - if dbBlock.Orphaned { + if dbBlock.Orphaned == 1 { slotData.Status = 2 } else { slotData.Status = 1 diff --git a/indexer/cache_logic.go b/indexer/cache_logic.go index af1b8122..533a9fcc 100644 --- a/indexer/cache_logic.go +++ b/indexer/cache_logic.go @@ -238,7 +238,7 @@ func (cache *indexerCache) processOrphanedBlocks(processedEpoch int64) error { continue } dbBlock := buildDbBlock(block, cache.getEpochStats(utils.EpochOfSlot(block.Slot), nil)) - dbBlock.Orphaned = true + dbBlock.Orphaned = 1 db.InsertBlock(dbBlock, tx) db.InsertOrphanedBlock(block.buildOrphanedBlock(), tx) } diff --git a/indexer/client.go b/indexer/client.go index 611675bd..8251c27c 100644 --- a/indexer/client.go +++ b/indexer/client.go @@ -101,7 +101,6 @@ func (client *IndexerClient) runIndexerClientLoop() { genesisTime := time.Unix(int64(utils.Config.Chain.GenesisTimestamp), 0) genesisSince := time.Since(genesisTime) - fmt.Printf("genesis %v\n", int(genesisSince.Seconds())) if genesisSince < 0 { if genesisSince > (time.Duration)(0-waitTime)*time.Second { waitTime = int(genesisSince.Abs().Seconds()) diff --git a/indexer/indexer.go b/indexer/indexer.go index 823aec90..a3ef57cd 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -485,6 +485,10 @@ func (indexer *Indexer) BuildLiveBlock(block *CacheBlock) *dbtypes.Block { } block.dbBlockCache = buildDbBlock(block, epochStats) } - block.dbBlockCache.Orphaned = !block.IsCanonical(indexer, nil) + if block.IsCanonical(indexer, nil) { + block.dbBlockCache.Orphaned = 0 + } else { + block.dbBlockCache.Orphaned = 1 + } return block.dbBlockCache } diff --git a/services/beaconservice.go b/services/beaconservice.go index d258a249..ebdc8e60 100644 --- a/services/beaconservice.go +++ b/services/beaconservice.go @@ -49,12 +49,7 @@ func StartBeaconService() error { } validatorNames := &ValidatorNames{} - if utils.Config.Frontend.ValidatorNamesYaml != "" { - validatorNames.LoadFromYaml(utils.Config.Frontend.ValidatorNamesYaml) - } - if utils.Config.Frontend.ValidatorNamesInventory != "" { - validatorNames.LoadFromRangesApi(utils.Config.Frontend.ValidatorNamesInventory) - } + validatorNames.LoadValidatorNames() GlobalBeaconService = &BeaconService{ indexer: indexer, @@ -490,104 +485,67 @@ func (bs *BeaconService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, return resBlocks } -func (bs *BeaconService) GetDbBlocksByGraffiti(graffiti string, pageIdx uint64, pageSize uint32, withOrphaned bool) []*dbtypes.Block { - cachedMatches := make([]*indexer.CacheBlock, 0) +type cachedDbBlock struct { + slot uint64 + proposer uint64 + block *indexer.CacheBlock +} + +func (bs *BeaconService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx uint64, pageSize uint32) []*dbtypes.AssignedBlock { + cachedMatches := make([]cachedDbBlock, 0) finalizedEpoch, _ := bs.indexer.GetFinalizedEpoch() idxMinSlot := (finalizedEpoch + 1) * int64(utils.Config.Chain.Config.SlotsPerEpoch) idxHeadSlot := bs.indexer.GetHighestSlot() - if idxMinSlot >= 0 { - for slotIdx := int64(idxHeadSlot); slotIdx >= int64(idxMinSlot); slotIdx-- { - slot := uint64(slotIdx) - blocks := bs.indexer.GetCachedBlocks(slot) - if blocks != nil { - for bidx := 0; bidx < len(blocks); bidx++ { - block := blocks[bidx] - if !withOrphaned && !block.IsCanonical(bs.indexer, nil) { + proposedMap := map[uint64]bool{} + for slotIdx := int64(idxHeadSlot); slotIdx >= int64(idxMinSlot); slotIdx-- { + slot := uint64(slotIdx) + blocks := bs.indexer.GetCachedBlocks(slot) + if blocks != nil { + for bidx := 0; bidx < len(blocks); bidx++ { + block := blocks[bidx] + if filter.WithOrphaned != 1 { + isOrphaned := !block.IsCanonical(bs.indexer, nil) + if filter.WithOrphaned == 0 && isOrphaned { continue } - blockGraffiti := string(block.GetBlockBody().Message.Body.Graffiti) - if !strings.Contains(blockGraffiti, graffiti) { + if filter.WithOrphaned == 2 && !isOrphaned { continue } - cachedMatches = append(cachedMatches, block) } - } - } - } - - cachedMatchesLen := uint64(len(cachedMatches)) - cachedPages := cachedMatchesLen / uint64(pageSize) - resBlocks := make([]*dbtypes.Block, 0) - resIdx := 0 - - cachedStart := pageIdx * uint64(pageSize) - cachedEnd := cachedStart + uint64(pageSize) - if cachedEnd+1 < cachedMatchesLen { - cachedEnd++ - } - - if cachedPages > 0 && pageIdx < cachedPages { - for _, block := range cachedMatches[cachedStart:cachedEnd] { - resBlocks = append(resBlocks, bs.indexer.BuildLiveBlock(block)) - resIdx++ - } - } else if pageIdx == cachedPages { - start := pageIdx * uint64(pageSize) - for _, block := range cachedMatches[start:] { - resBlocks = append(resBlocks, bs.indexer.BuildLiveBlock(block)) - resIdx++ - } - } - if resIdx > int(pageSize) { - return resBlocks - } - - // load from db - var dbMinSlot uint64 - if idxMinSlot < 0 { - dbMinSlot = utils.TimeToSlot(uint64(time.Now().Unix())) - } else { - dbMinSlot = uint64(idxMinSlot) - } - - dbPage := pageIdx - cachedPages - dbCacheOffset := uint64(pageSize) - (cachedMatchesLen % uint64(pageSize)) - var dbBlocks []*dbtypes.Block - if dbPage == 0 { - dbBlocks = db.GetBlocksWithGraffiti(graffiti, dbMinSlot, 0, uint32(dbCacheOffset)+1, withOrphaned) - } else { - dbBlocks = db.GetBlocksWithGraffiti(graffiti, dbMinSlot, (dbPage-1)*uint64(pageSize)+dbCacheOffset, pageSize+1, withOrphaned) - } - resBlocks = append(resBlocks, dbBlocks...) - - return resBlocks -} + proposedMap[block.Slot] = true + if filter.WithMissing == 2 { + continue + } -func (bs *BeaconService) GetDbBlocksByProposer(proposer uint64, pageIdx uint64, pageSize uint32, withMissing bool, withOrphaned bool) []*dbtypes.AssignedBlock { - cachedMatches := make([]struct { - slot uint64 - block *indexer.CacheBlock - }, 0) - finalizedEpoch, _ := bs.indexer.GetFinalizedEpoch() - idxMinSlot := (finalizedEpoch + 1) * int64(utils.Config.Chain.Config.SlotsPerEpoch) + if filter.Graffiti != "" { + blockGraffiti := string(block.GetBlockBody().Message.Body.Graffiti) + if !strings.Contains(blockGraffiti, filter.Graffiti) { + continue + } + } + proposer := uint64(block.GetBlockBody().Message.ProposerIndex) + if filter.ProposerIndex != nil { + if proposer != *filter.ProposerIndex { + continue + } + } + if filter.ProposerName != "" { + proposerName := bs.validatorNames.GetValidatorName(proposer) + if !strings.Contains(proposerName, filter.ProposerName) { + continue + } + } - // get proposed blocks - proposedMap := map[uint64]bool{} - for _, block := range bs.indexer.GetCachedBlocksByProposer(proposer) { - if !withOrphaned && !block.IsCanonical(bs.indexer, nil) { - continue + cachedMatches = append(cachedMatches, cachedDbBlock{ + slot: block.Slot, + proposer: uint64(block.GetBlockBody().Message.ProposerIndex), + block: block, + }) + } } - proposedMap[block.Slot] = true - cachedMatches = append(cachedMatches, struct { - slot uint64 - block *indexer.CacheBlock - }{ - slot: block.Slot, - block: block, - }) } - if withMissing { + if filter.WithMissing != 0 && filter.Graffiti == "" && filter.WithOrphaned != 2 { // add missed blocks idxHeadSlot := bs.indexer.GetHighestSlot() idxHeadEpoch := utils.EpochOfSlot(idxHeadSlot) @@ -599,18 +557,26 @@ func (bs *BeaconService) GetDbBlocksByProposer(proposer uint64, pageIdx uint64, continue } for slot, assigned := range epochStats.GetProposerAssignments() { - if assigned != proposer { - continue - } if proposedMap[slot] { continue } - cachedMatches = append(cachedMatches, struct { - slot uint64 - block *indexer.CacheBlock - }{ - slot: slot, - block: nil, + + if filter.ProposerIndex != nil { + if assigned != *filter.ProposerIndex { + continue + } + } + if filter.ProposerName != "" { + assignedName := bs.validatorNames.GetValidatorName(assigned) + if assignedName == "" || !strings.Contains(assignedName, filter.ProposerName) { + continue + } + } + + cachedMatches = append(cachedMatches, cachedDbBlock{ + slot: slot, + proposer: assigned, + block: nil, }) } sort.Slice(cachedMatches, func(a, b int) bool { @@ -636,13 +602,12 @@ func (bs *BeaconService) GetDbBlocksByProposer(proposer uint64, pageIdx uint64, for _, block := range cachedMatches[cachedStart:cachedEnd] { assignedBlock := dbtypes.AssignedBlock{ Slot: block.slot, - Proposer: proposer, + Proposer: block.proposer, } if block.block != nil { assignedBlock.Block = bs.indexer.BuildLiveBlock(block.block) } resBlocks = append(resBlocks, &assignedBlock) - resIdx++ } } else if pageIdx == cachedPages { @@ -650,7 +615,7 @@ func (bs *BeaconService) GetDbBlocksByProposer(proposer uint64, pageIdx uint64, for _, block := range cachedMatches[start:] { assignedBlock := dbtypes.AssignedBlock{ Slot: block.slot, - Proposer: proposer, + Proposer: block.proposer, } if block.block != nil { assignedBlock.Block = bs.indexer.BuildLiveBlock(block.block) @@ -675,9 +640,9 @@ func (bs *BeaconService) GetDbBlocksByProposer(proposer uint64, pageIdx uint64, dbCacheOffset := uint64(pageSize) - (cachedMatchesLen % uint64(pageSize)) var dbBlocks []*dbtypes.AssignedBlock if dbPage == 0 { - dbBlocks = db.GetAssignedBlocks(proposer, dbMinSlot, 0, uint32(dbCacheOffset)+1, withOrphaned) + dbBlocks = db.GetFilteredBlocks(filter, dbMinSlot, 0, uint32(dbCacheOffset)+1) } else { - dbBlocks = db.GetAssignedBlocks(proposer, dbMinSlot, (dbPage-1)*uint64(pageSize)+dbCacheOffset, pageSize+1, withOrphaned) + dbBlocks = db.GetFilteredBlocks(filter, dbMinSlot, (dbPage-1)*uint64(pageSize)+dbCacheOffset, pageSize+1) } resBlocks = append(resBlocks, dbBlocks...) diff --git a/services/validatornames.go b/services/validatornames.go index 6e31070f..caa9b968 100644 --- a/services/validatornames.go +++ b/services/validatornames.go @@ -6,22 +6,32 @@ import ( "io" "net/http" "os" + "sort" "strconv" "strings" "sync" "time" + "github.com/pk910/light-beaconchain-explorer/db" + "github.com/pk910/light-beaconchain-explorer/dbtypes" + "github.com/pk910/light-beaconchain-explorer/utils" "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) +var logger_vn = logrus.StandardLogger().WithField("module", "validator_names") + type ValidatorNames struct { - namesMutex sync.RWMutex - names map[uint64]string + loadingMutex sync.Mutex + loading bool + namesMutex sync.RWMutex + names map[uint64]string } func (vn *ValidatorNames) GetValidatorName(index uint64) string { - vn.namesMutex.RLock() + if !vn.namesMutex.TryRLock() { + return "" + } defer vn.namesMutex.RUnlock() if vn.names == nil { return "" @@ -29,13 +39,45 @@ func (vn *ValidatorNames) GetValidatorName(index uint64) string { return vn.names[index] } -func (vn *ValidatorNames) LoadFromYaml(fileName string) error { - vn.namesMutex.Lock() - defer vn.namesMutex.Unlock() +func (vn *ValidatorNames) LoadValidatorNames() { + vn.loadingMutex.Lock() + defer vn.loadingMutex.Unlock() + if vn.loading { + return + } + vn.loading = true + + go func() { + vn.namesMutex.Lock() + vn.names = make(map[uint64]string) + vn.namesMutex.Unlock() + + // load names + if utils.Config.Frontend.ValidatorNamesYaml != "" { + err := vn.loadFromYaml(utils.Config.Frontend.ValidatorNamesYaml) + if err != nil { + logger_vn.WithError(err).Errorf("error while loading validator names from yaml") + } + } + if utils.Config.Frontend.ValidatorNamesInventory != "" { + err := vn.loadFromRangesApi(utils.Config.Frontend.ValidatorNamesInventory) + if err != nil { + logger_vn.WithError(err).Errorf("error while loading validator names inventory") + } + } + // update db + if !utils.Config.Indexer.DisableIndexWriter { + vn.updateDb() + } + + vn.loading = false + }() +} + +func (vn *ValidatorNames) loadFromYaml(fileName string) error { f, err := os.Open(fileName) if err != nil { - logrus.Errorf("error opening validator names file %v: %v", fileName, err) return fmt.Errorf("error opening validator names file %v: %v", fileName, err) } @@ -43,14 +85,12 @@ func (vn *ValidatorNames) LoadFromYaml(fileName string) error { decoder := yaml.NewDecoder(f) err = decoder.Decode(&namesYaml) if err != nil { - logrus.Errorf("error decoding validator names file %v: %v", fileName, err) return fmt.Errorf("error decoding validator names file %v: %v", fileName, err) } + vn.namesMutex.Lock() + defer vn.namesMutex.Unlock() nameCount := 0 - if vn.names == nil { - vn.names = make(map[uint64]string) - } for idxStr, name := range namesYaml { rangeParts := strings.Split(idxStr, "-") minIdx, err := strconv.ParseUint(rangeParts[0], 10, 64) @@ -69,7 +109,7 @@ func (vn *ValidatorNames) LoadFromYaml(fileName string) error { nameCount++ } } - logrus.Infof("Loaded %v validator names from yaml (%v)", nameCount, fileName) + logger_vn.Infof("loaded %v validator names from yaml (%v)", nameCount, fileName) return nil } @@ -78,25 +118,22 @@ type validatorNamesRangesResponse struct { Ranges map[string]string `json:"ranges"` } -func (vn *ValidatorNames) LoadFromRangesApi(apiUrl string) error { - vn.namesMutex.Lock() - defer vn.namesMutex.Unlock() - logrus.Debugf("Loading validator names from inventory: %v", apiUrl) +func (vn *ValidatorNames) loadFromRangesApi(apiUrl string) error { + logger_vn.Debugf("Loading validator names from inventory: %v", apiUrl) client := &http.Client{Timeout: time.Second * 120} resp, err := client.Get(apiUrl) if err != nil { - logrus.Errorf("Could not fetch validator names from inventory (%v): %v", apiUrl, err) - return err + return fmt.Errorf("could not fetch inventory (%v): %v", utils.GetRedactedUrl(apiUrl), err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { - logrus.Errorf("Could not fetch validator names from inventory (%v): not found", apiUrl) + logger_vn.Errorf("could not fetch inventory (%v): not found", utils.GetRedactedUrl(apiUrl)) return nil } data, _ := io.ReadAll(resp.Body) - return fmt.Errorf("url: %v, error-response: %s", apiUrl, data) + return fmt.Errorf("url: %v, error-response: %s", utils.GetRedactedUrl(apiUrl), data) } rangesResponse := &validatorNamesRangesResponse{} dec := json.NewDecoder(resp.Body) @@ -105,9 +142,8 @@ func (vn *ValidatorNames) LoadFromRangesApi(apiUrl string) error { return fmt.Errorf("error parsing validator ranges response: %v", err) } - if vn.names == nil { - vn.names = make(map[uint64]string) - } + vn.namesMutex.Lock() + defer vn.namesMutex.Unlock() nameCount := 0 for rangeStr, name := range rangesResponse.Ranges { rangeParts := strings.Split(rangeStr, "-") @@ -127,6 +163,88 @@ func (vn *ValidatorNames) LoadFromRangesApi(apiUrl string) error { nameCount++ } } - logrus.Infof("Loaded %v validator names from inventory api (%v)", nameCount, apiUrl) + logger_vn.Infof("loaded %v validator names from inventory api (%v)", nameCount, utils.GetRedactedUrl(apiUrl)) + return nil +} + +func (vn *ValidatorNames) updateDb() error { + vn.namesMutex.RLock() + nameRows := make([]*dbtypes.ValidatorName, 0) + for index, name := range vn.names { + nameRows = append(nameRows, &dbtypes.ValidatorName{ + Index: index, + Name: name, + }) + } + vn.namesMutex.RUnlock() + + sort.Slice(nameRows, func(a, b int) bool { + return nameRows[a].Index < nameRows[b].Index + }) + + tx, err := db.WriterDb.Beginx() + if err != nil { + return fmt.Errorf("error starting db transaction: %v", err) + } + defer tx.Rollback() + + batchSize := 10000 + + lastIndex := uint64(0) + nameIdx := 0 + nameLen := len(nameRows) + for nameIdx < nameLen { + maxIdx := nameIdx + batchSize + if maxIdx >= nameLen { + maxIdx = nameLen + } + sliceLen := maxIdx - nameIdx + namesSlice := nameRows[nameIdx:maxIdx] + maxIndex := namesSlice[sliceLen-1].Index + + // get existing db entries + dbNamesMap := map[uint64]string{} + for _, dbName := range db.GetValidatorNames(lastIndex, maxIndex, tx) { + dbNamesMap[dbName.Index] = dbName.Name + } + + // get diffs + updateNames := make([]*dbtypes.ValidatorName, 0) + for _, nameRow := range namesSlice { + dbName := dbNamesMap[nameRow.Index] + delete(dbNamesMap, nameRow.Index) + if dbName == nameRow.Name { + continue // no update + } + updateNames = append(updateNames, nameRow) + } + + removeIndexes := make([]uint64, 0) + for index := range dbNamesMap { + removeIndexes = append(removeIndexes, index) + } + + if len(updateNames) > 0 { + err := db.InsertValidatorNames(updateNames, tx) + if err != nil { + logger_vn.WithError(err).Errorf("error while adding validator names to db") + } + } + if len(removeIndexes) > 0 { + err := db.DeleteValidatorNames(removeIndexes, tx) + if err != nil { + logger_vn.WithError(err).Errorf("error while deleting validator names from db") + } + } + logger_vn.Debugf("update validator names %v-%v: %v changed, %v removed", lastIndex, maxIdx, len(updateNames), len(removeIndexes)) + + lastIndex = maxIndex + 1 + nameIdx = maxIdx + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("error committing db transaction: %v", err) + } + return nil } diff --git a/static/js/explorer.js b/static/js/explorer.js index aba260f5..5ede3167 100644 --- a/static/js/explorer.js +++ b/static/js/explorer.js @@ -221,7 +221,7 @@ // sug.graffiti is html-escaped to prevent xss, we need to unescape it var el = document.createElement("textarea") el.innerHTML = sug.graffiti - window.location = "/slots?q=" + encodeURIComponent(el.value) + window.location = "/slots/filtered?f&f.graffiti=" + encodeURIComponent(el.value) } else { console.log("invalid typeahead-selection", sug) } diff --git a/templates/epochs/epochs.html b/templates/epochs/epochs.html index 8d288e04..46820ad9 100644 --- a/templates/epochs/epochs.html +++ b/templates/epochs/epochs.html @@ -13,7 +13,6 @@

-
diff --git a/templates/slots/slots.html b/templates/slots/slots.html index 508babba..2d596c84 100644 --- a/templates/slots/slots.html +++ b/templates/slots/slots.html @@ -11,7 +11,6 @@

Slots

-
@@ -28,21 +27,15 @@

Slots

{{ if not .IsDefaultPage }} {{ end }} - {{ if .GraffitiFilter }} - - {{ end }} entries
-
- -
+ + Filter Blocks +
@@ -50,9 +43,7 @@

Slots

- {{ if .ShowForkTree -}} - - {{- end }} + @@ -74,26 +65,23 @@

Slots

{{ if gt .SlotCount 0 }} - {{ $showTree := .ShowForkTree }} {{ $treeWidth := .ForkTreeWidth }} {{ range $i, $slot := .Slots }} - {{ if $showTree -}} - - {{- end }} + {{ if eq $slot.Status 2 }} @@ -159,19 +147,19 @@

Slots

diff --git a/templates/slots_filtered/slots_filtered.html b/templates/slots_filtered/slots_filtered.html new file mode 100644 index 00000000..d3497efd --- /dev/null +++ b/templates/slots_filtered/slots_filtered.html @@ -0,0 +1,232 @@ +{{ define "page" }} +
+
+

Filtered Slots

+ +
+ +
+
+ +
+
+ Slot Filters +
+
+
+
+
+
+
+ Graffiti +
+
+ +
+
+
+
+ Proposer Index +
+
+ +
+
+
+
+ Proposer Name +
+
+ +
+
+
+
+
+
+
+
+ Missing Blocks +
+
+ +
+
+
+
+ Orphaned Blocks +
+
+ +
+
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+
+
+ + + +
+
+ +
+
ChainChain Epoch Slot Status
- {{ range $j, $graph := $slot.ForkGraph }} -
- {{- range $tile, $val := $graph.Tiles -}} -
- {{- end -}} - {{- if $graph.Block }} -
- -
- {{ end -}} -
- {{ end }} - 		+ {{ range $j, $graph := $slot.ForkGraph }} +
+ {{- range $tile, $val := $graph.Tiles -}} +
+ {{- end -}} + {{- if $graph.Block }} +
+ +
+ {{ end -}} +
+ {{ end }} + 		{{ formatAddCommas $slot.Epoch }}{{ formatAddCommas $slot.Slot }}
+ + + + + + + + + + + + + + + + {{ if gt .SlotCount 0 }} + + {{ range $i, $slot := .Slots }} + + + {{ if eq $slot.Status 2 }} + + {{ else }} + + {{ end }} + + + {{ if $slot.Synchronized }} + + + + + + + + {{ else }} + + {{ end }} + + + {{ end }} + + {{ else }} + + + + + + + + {{ end }} +
EpochSlotStatusTimeProposerAttestations + Deposits / + Exits + Slashings + P / + A + Tx CountSync Agg %Graffiti
{{ formatAddCommas $slot.Epoch }}{{ formatAddCommas $slot.Slot }}{{ formatAddCommas $slot.Slot }} + {{ if eq $slot.Slot 0 }} + Genesis + {{ else if eq $slot.Status 1 }} + Proposed + {{ else if eq $slot.Status 2 }} + Orphaned + {{ else if $slot.Scheduled }} + Scheduled + {{ else if not $slot.Synchronized }} + ? + {{ else if eq $slot.Status 0 }} + Missed + {{ else }} + Unknown + {{ end }} + {{ formatRecentTimeShort $slot.Ts }}{{ formatValidator $slot.Proposer $slot.ProposerName }}{{ if not (eq $slot.Status 0) }}{{ $slot.AttestationCount }}{{ end }}{{ if not (eq $slot.Status 0) }}{{ $slot.DepositCount }} / {{ $slot.ExitCount }}{{ end }}{{ if not (eq $slot.Status 0) }}{{ $slot.ProposerSlashingCount }} / {{ $slot.AttesterSlashingCount }}{{ end }}{{ if not (eq $slot.Status 0) }}{{ $slot.EthTransactionCount }}{{ end }}{{ if not (eq $slot.Status 0) }}{{ formatFloat $slot.SyncParticipation 2 }}%{{ end }}{{ if not (eq $slot.Status 0) }}{{ formatGraffiti $slot.Graffiti }}{{ end }}Not indexed yet
+
+ {{ template "professor_svg" }} +
+
+
+ {{ if gt .TotalPages 1 }} +
+
+
+
Showing slot {{ .FirstSlot }} to {{ .LastSlot }}
+
+
+
+
+ +
+
+
+ {{ end }} +
+ +
+
+{{ end }} +{{ define "js" }} +{{ end }} +{{ define "css" }} +{{ end }} \ No newline at end of file diff --git a/templates/validators/validators.html b/templates/validators/validators.html index 0776a83f..1bad87df 100644 --- a/templates/validators/validators.html +++ b/templates/validators/validators.html @@ -12,7 +12,6 @@

Validators Overvie

-
diff --git a/types/models/slots.go b/types/models/slots.go index 4416eeb0..7774b589 100644 --- a/types/models/slots.go +++ b/types/models/slots.go @@ -6,13 +6,11 @@ import ( // SlotsPageData is a struct to hold info for the slots page type SlotsPageData struct { - Slots []*SlotsPageDataSlot `json:"slots"` - SlotCount uint64 `json:"slot_count"` - FirstSlot uint64 `json:"first_slot"` - LastSlot uint64 `json:"last_slot"` - ShowForkTree bool `json:"show_forktree"` - ForkTreeWidth int `json:"forktree_width"` - GraffitiFilter string `json:"graffiti_filter"` + Slots []*SlotsPageDataSlot `json:"slots"` + SlotCount uint64 `json:"slot_count"` + FirstSlot uint64 `json:"first_slot"` + LastSlot uint64 `json:"last_slot"` + ForkTreeWidth int `json:"forktree_width"` IsDefaultPage bool `json:"default_page"` TotalPages uint64 `json:"total_pages"` diff --git a/types/models/slots_filtered.go b/types/models/slots_filtered.go new file mode 100644 index 00000000..50b8af44 --- /dev/null +++ b/types/models/slots_filtered.go @@ -0,0 +1,58 @@ +package models + +import ( + "time" +) + +// SlotsPageData is a struct to hold info for the slots page +type SlotsFilteredPageData struct { + FilterGraffiti string `json:"filter_graffiti"` + FilterProposer string `json:"filter_proposer"` + FilterProposerName string `json:"filter_pname"` + FilterWithOrphaned uint8 `json:"filter_orphaned"` + FilterWithMissing uint8 `json:"filter_missing"` + + Slots []*SlotsFilteredPageDataSlot `json:"slots"` + SlotCount uint64 `json:"slot_count"` + FirstSlot uint64 `json:"first_slot"` + LastSlot uint64 `json:"last_slot"` + + IsDefaultPage bool `json:"default_page"` + TotalPages uint64 `json:"total_pages"` + PageSize uint64 `json:"page_size"` + CurrentPageIndex uint64 `json:"page_index"` + CurrentPageSlot uint64 `json:"page_slot"` + PrevPageIndex uint64 `json:"prev_page_index"` + PrevPageSlot uint64 `json:"prev_page_slot"` + NextPageIndex uint64 `json:"next_page_index"` + NextPageSlot uint64 `json:"next_page_slot"` + LastPageSlot uint64 `json:"last_page_slot"` + + FirstPageLink string `json:"first_page_link"` + PrevPageLink string `json:"prev_page_link"` + NextPageLink string `json:"next_page_link"` + LastPageLink string `json:"last_page_link"` +} + +type SlotsFilteredPageDataSlot struct { + Slot uint64 `json:"slot"` + Epoch uint64 `json:"epoch"` + Ts time.Time `json:"ts"` + Finalized bool `json:"scheduled"` + Scheduled bool `json:"finalized"` + Status uint8 `json:"status"` + Synchronized bool `json:"synchronized"` + Proposer uint64 `json:"proposer"` + ProposerName string `json:"proposer_name"` + AttestationCount uint64 `json:"attestation_count"` + DepositCount uint64 `json:"deposit_count"` + ExitCount uint64 `json:"exit_count"` + ProposerSlashingCount uint64 `json:"proposer_slashing_count"` + AttesterSlashingCount uint64 `json:"attester_slashing_count"` + SyncParticipation float64 `json:"sync_participation"` + EthTransactionCount uint64 `json:"eth_transaction_count"` + EthBlockNumber uint64 `json:"eth_block_number"` + Graffiti []byte `json:"graffiti"` + BlockRoot []byte `json:"block_root"` + ParentRoot []byte `json:"parent_root"` +}