feat: daemon service for collecting metrics and state

Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 
 ADD . .
 
-RUN CGO_ENABLED=0 go build -o ./bin/algorun *.go
+RUN CGO_ENABLED=0 go build -o ./bin/algorun main.go && CGO_ENABLED=0 go build -o ./bin/fortiter daemon/main.go
 
 FROM algorand/algod:latest
 
@@ -20,6 +20,7 @@ ADD .docker/start_empty.sh /node/run/start_empty.sh
 ADD .docker/start_fast_catchup.sh /node/run/start_fast_catchup.sh
 
 COPY --from=builder /app/bin/algorun /bin/algorun
+COPY --from=BUILDER /app/bin/fortiter /bin/fortiter
 
 RUN apt-get update && apt-get install jq -y
 

Makefile

@@ -1,5 +1,7 @@
 build:
-	CGO_ENABLED=0 go build -o bin/algorun *.go
+	CGO_ENABLED=0 go build -o bin/algorun main.go
+build-daemon:
+	CGO_ENABLED=0 go build -o bin/fortiter -ldflags "-X cmd.version=testing" daemon/main.go 
 test:
 	go test -coverpkg=./... -covermode=atomic ./...
 generate:

cmd/root.go

@@ -7,7 +7,7 @@ import (
 	"github.com/algorandfoundation/algorun-tui/api"
 	"github.com/algorandfoundation/algorun-tui/cmd/configure"
 	"github.com/algorandfoundation/algorun-tui/cmd/node"
-	"github.com/algorandfoundation/algorun-tui/internal"
+	"github.com/algorandfoundation/algorun-tui/internal/nodekit"
 	"github.com/algorandfoundation/algorun-tui/ui"
 	"github.com/algorandfoundation/algorun-tui/ui/explanations"
 	"github.com/algorandfoundation/algorun-tui/ui/style"
@@ -64,23 +64,23 @@ var (
 					v.StatusCode())
 			}
 
-			partkeys, err := internal.GetPartKeys(ctx, client)
+			partkeys, err := nodekit.GetPartKeys(ctx, client)
 			if err != nil {
 				return fmt.Errorf(
 					style.Red.Render("failed to get participation keys: %s")+
 						explanations.TokenNotAdmin,
 					err)
 			}
-			state := internal.StateModel{
-				Status: internal.StatusModel{
+			state := nodekit.StateModel{
+				Status: nodekit.StatusModel{
 					State:       "INITIALIZING",
 					Version:     "N/A",
 					Network:     "N/A",
 					Voting:      false,
 					NeedsUpdate: true,
 					LastRound:   0,
 				},
-				Metrics: internal.MetricsModel{
+				Metrics: nodekit.MetricsModel{
 					RoundTime: 0,
 					TPS:       0,
 					RX:        0,
@@ -91,10 +91,10 @@ var (
 				Client:  client,
 				Context: ctx,
 			}
-			state.Accounts, err = internal.AccountsFromState(&state, new(internal.Clock), client)
+			state.Accounts, err = nodekit.AccountsFromState(&state, new(nodekit.Clock), client)
 			cobra.CheckErr(err)
 			// Fetch current state
-			err = state.Status.Fetch(ctx, client, new(internal.HttpPkg))
+			err = state.Status.Fetch(ctx, client, new(nodekit.HttpPkg))
 			cobra.CheckErr(err)
 
 			m, err := ui.NewViewportViewModel(&state, client)
@@ -106,7 +106,7 @@ var (
 				tea.WithFPS(120),
 			)
 			go func() {
-				state.Watch(func(status *internal.StateModel, err error) {
+				state.Watch(func(status *nodekit.StateModel, err error) {
 					if err == nil {
 						p.Send(state)
 					}

cmd/status.go

@@ -4,7 +4,7 @@
 	"context"
 	"errors"
 	"fmt"
-	"github.com/algorandfoundation/algorun-tui/internal"
+	"github.com/algorandfoundation/algorun-tui/internal/nodekit"
 	"github.com/algorandfoundation/algorun-tui/ui"
 	"github.com/algorandfoundation/algorun-tui/ui/style"
 	tea "github.com/charmbracelet/bubbletea"
@@ -27,31 +27,31 @@
 		// Get Algod from configuration
 		client, err := getClient()
 		cobra.CheckErr(err)
-		state := internal.StateModel{
-			Status: internal.StatusModel{
+		state := nodekit.StateModel{
+			Status: nodekit.StatusModel{
 				State:       "SYNCING",
 				Version:     "N/A",
 				Network:     "N/A",
 				Voting:      false,
 				NeedsUpdate: true,
 				LastRound:   0,
 			},
-			Metrics: internal.MetricsModel{
+			Metrics: nodekit.MetricsModel{
 				RoundTime: 0,
 				TPS:       0,
 				RX:        0,
 				TX:        0,
 			},
 			ParticipationKeys: nil,
 		}
-		err = state.Status.Fetch(context.Background(), client, new(internal.HttpPkg))
+		err = state.Status.Fetch(context.Background(), client, new(nodekit.HttpPkg))
 		cobra.CheckErr(err)
 		// Create the TUI
 		view := ui.MakeStatusViewModel(&state)
 
 		p := tea.NewProgram(view, tea.WithAltScreen())
 		go func() {
-			state.Watch(func(status *internal.StateModel, err error) {
+			state.Watch(func(status *nodekit.StateModel, err error) {
 				cobra.CheckErr(err)
 				p.Send(state)
 			}, context.Background(), client)

daemon/cmd/root.go

@@ -0,0 +1,204 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/algorand/go-algorand/config"
+	"github.com/algorandfoundation/algorun-tui/daemon/fortiter"
+	"github.com/algorandfoundation/algorun-tui/daemon/rpc"
+	"github.com/algorandfoundation/algorun-tui/ui/style"
+	"github.com/charmbracelet/log"
+	"github.com/jmoiron/sqlx"
+	"github.com/labstack/echo-contrib/echoprometheus"
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"io"
+	"os"
+	"strings"
+)
+
+const BANNER = `
+ ______   ______     ______     ______   __     ______   ______     ______    
+/\  ___\ /\  __ \   /\  == \   /\__  _\ /\ \   /\__  _\ /\  ___\   /\  == \   
+\ \  __\ \ \ \/\ \  \ \  __<   \/_/\ \/ \ \ \  \/_/\ \/ \ \  __\   \ \  __<   
+ \ \_\    \ \_____\  \ \_\ \_\    \ \_\  \ \_\    \ \_\  \ \_____\  \ \_\ \_\ 
+  \/_/     \/_____/   \/_/ /_/     \/_/   \/_/     \/_/   \/_____/   \/_/ /_/ 
+`
+
+var version = ""
+var (
+	Version = version
+	sqlFile string
+	rootCmd = &cobra.Command{
+		Version: Version,
+		Use:     "fortiter",
+		Short:   "Consume all the data",
+		Long:    style.Purple(BANNER) + "\n",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			log.SetOutput(cmd.OutOrStdout())
+
+			e := echo.New()
+			e.HideBanner = true
+
+			// TODO: handle user interaction
+			e.Use(middleware.Logger())
+			e.Use(middleware.Recover())
+			e.Use(echoprometheus.NewMiddleware("fortiter"))
+			e.Static("/", "public")
+
+			fmt.Println(style.Magenta(BANNER))
+			fmt.Println(style.LightBlue("Database: ") + viper.GetString("database"))
+
+			algodConfig := viper.GetString("data")
+			if algodConfig != "" {
+				fmt.Println(style.LightBlue("Configuration: ") + algodConfig)
+			}
+			logFile := viper.GetString("log")
+			if logFile != "" {
+				fmt.Println(style.LightBlue("Log: ") + logFile)
+			}
+			var si = fortiter.Handlers{PrometheusHandler: echoprometheus.NewHandler()}
+			rpc.RegisterHandlers(e, si)
+
+			db, err := sqlx.Connect("sqlite3", viper.GetString("database"))
+			cobra.CheckErr(err)
+
+			// exec the schema or fail; multi-statement Exec behavior varies between
+			// database drivers;  pq will exec them all, sqlite3 won't, ymmv
+			db.MustExec(fortiter.Schema)
+			db.MustExec(fortiter.StatsSchema)
+
+			err = fortiter.Sync(context.Background(), logFile, db)
+			cobra.CheckErr(err)
+
+			return e.Start(":1337")
+		},
+	}
+)
+
+func check(err interface{}) {
+	if err != nil {
+		log.Fatal(err)
+		panic(err)
+	}
+}
+
+// Handle global flags and set usage templates
+func init() {
+	log.SetReportTimestamp(false)
+	initConfig()
+	// Configure Version
+	if Version == "" {
+		Version = "unknown (built from source)"
+	}
+	rootCmd.Version = Version
+
+	// Bindings
+	rootCmd.PersistentFlags().StringVar(&sqlFile, "database", "fortiter.db", style.LightBlue("database file location"))
+	_ = viper.BindPFlag("database", rootCmd.PersistentFlags().Lookup("database"))
+	// Update Long Text
+	rootCmd.Long +=
+		//style.Magenta("Database: ") + viper.GetViper().ConfigFileUsed() + "\n" +
+		style.LightBlue("Database: ") + viper.GetString("database")
+
+	if viper.GetString("data") != "" {
+		rootCmd.Long +=
+			style.Magenta("\nAlgorand Data: ") + viper.GetString("data")
+	}
+}
+
+// Execute executes the root command.
+func Execute() error {
+	return rootCmd.Execute()
+}
+
+type AlgodConfig struct {
+	EndpointAddress string `json:"EndpointAddress"`
+}
+
+func initExistingAlgod() {
+	algorandData, exists := os.LookupEnv("ALGORAND_DATA")
+
+	// Load the Algorand Data Configuration
+	if exists && algorandData != "" {
+		// Placeholder for Struct
+		var algodConfig config.Local
+
+		dataConfigPath := algorandData + "/config.json"
+
+		// Open the config.json File
+		configFile, err := os.Open(dataConfigPath)
+		check(err)
+
+		// Read the bytes of the File
+		byteValue, _ := io.ReadAll(configFile)
+		err = json.Unmarshal(byteValue, &algodConfig)
+		check(err)
+
+		// Close the open handle
+		err = configFile.Close()
+		check(err)
+
+		// Replace catchall address with localhost
+		if strings.Contains(algodConfig.EndpointAddress, "0.0.0.0") {
+			algodConfig.EndpointAddress = strings.Replace(algodConfig.EndpointAddress, "0.0.0.0", "127.0.0.1", 1)
+		}
+
+		// Handle Token Path
+		tokenPath := algorandData + "/algod.admin.token"
+
+		tokenFile, err := os.Open(tokenPath)
+		check(err)
+
+		byteValue, err = io.ReadAll(tokenFile)
+		check(err)
+	}
+}
+func initConfig() {
+	// Load ALGORAND_DATA/config.json
+	algorandData, exists := os.LookupEnv("ALGORAND_DATA")
+
+	// Load the Algorand Data Configuration
+	if exists && algorandData != "" {
+		// Placeholder for Struct
+		var algodConfig config.Local
+
+		dataConfigPath := algorandData + "/config.json"
+
+		// Open the config.json File
+		configFile, err := os.Open(dataConfigPath)
+		check(err)
+
+		// Read the bytes of the File
+		byteValue, _ := io.ReadAll(configFile)
+		err = json.Unmarshal(byteValue, &algodConfig)
+		check(err)
+
+		// Close the open handle
+		err = configFile.Close()
+		check(err)
+
+		// Find the log file
+		logPath, _ := algodConfig.ResolveLogPaths(algorandData)
+
+		// Handle Token Path
+		tokenPath := algorandData + "/algod.admin.token"
+
+		tokenFile, err := os.Open(tokenPath)
+		check(err)
+
+		byteValue, err = io.ReadAll(tokenFile)
+		check(err)
+
+		// Set the server configuration
+		viper.Set("server", "http://"+algodConfig.EndpointAddress)
+		viper.Set("token", string(byteValue))
+		viper.Set("data", dataConfigPath)
+		viper.Set("log", logPath)
+	}
+
+}

daemon/fortiter/agreements.go

@@ -0,0 +1,20 @@
+package fortiter
+
+import (
+	"github.com/algorand/go-algorand/logging/logspec"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/mattn/go-sqlite3"
+	"time"
+)
+
+type AgreementEvent struct {
+	logspec.AgreementEvent
+	Message string    `json:"msg"`
+	Time    time.Time `json:"time"`
+}
+
+func SaveAgreement(event AgreementEvent, db *sqlx.DB) error {
+	a := event.AgreementEvent
+	db.MustExec("INSERT INTO agreements (type, round, period, step, hash, sender, object_round, object_period, object_step, weight, weight_total, message, time) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", a.Type, a.Round, a.Period, a.Step, a.Hash, a.Sender, a.ObjectRound, a.ObjectPeriod, a.ObjectStep, a.Weight, a.WeightTotal, event.Message, event.Time)
+	return nil
+}