feat: add viper support to cobra

README.md

@@ -125,18 +125,38 @@ Extract blockchain data and output it to a PostgreSQL database.
 
 ```
 Usage:
-  yaci extract postgres [address] [psql-connection-string] [flags]
+  yaci extract postgres [address] [flags]
 ```
 
+#### Flags
+
+- `-p`, `--postgres-conn` - The PostgreSQL connection string
 
 #### Example
 
 ```shell
-yaci extract postgres localhost:9090 postgres://postgres:foobar@localhost/postgres -s 106000 -k --live -t 5
+yaci extract postgres localhost:9090 -p postgres://postgres:foobar@localhost/postgres -s 106000 -k --live -t 5
 ```
 
 This command will connect to the gRPC server running on `localhost:9090`, continuously extract data from block height `106000` and store the extracted data in the `postgres` database. New blocks and transactions will be inserted into the database every 5 seconds.
 
+## Configuration
+
+The `yaci` tool parameters can be configured from the following sources:
+
+- Environment variables (prefixed with `YACI_`)
+- Configuration file (`config.yaml`, `config.json`, `config.toml`, `config.hcl`, `config.env`)
+- Command-line flags
+
+The command-line flags have the highest priority, followed by the environment variables, and then the configuration file.
+
+The environment variables are prefixed with `YACI_` and are in uppercase. For example, the `--logLevel` flag can be set using the `YACI_LOGLEVEL` environment variable. Dash (`-`) is replaced with underscore (`_`). For example, the `--block-time` flag can be set using the `YACI_BLOCK_TIME` environment variable.
+
+The configuration file is searched in the following order:
+- The current working directory (`./`)
+- The user's home directory (`$HOME/.yaci`)
+- The system's configuration directory (`/etc/yaci`)
+
 ## Demo
 
 To run the demo, you need to have Docker installed on your system. Then, you can run the following command:

cmd/yaci/extract.go

@@ -9,37 +9,32 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"github.com/liftedinit/yaci/internal/client"
 	"github.com/liftedinit/yaci/internal/extractor"
 	"github.com/liftedinit/yaci/internal/output"
 	"github.com/liftedinit/yaci/internal/reflection"
 )
 
-var (
-	start          uint64
-	stop           uint64
-	insecure       bool
-	live           bool
-	blockTime      uint
-	maxConcurrency uint
-	maxRetries     uint
-)
-
 var ExtractCmd = &cobra.Command{
 	Use:   "extract",
 	Short: "Extract chain data to various output formats",
 	Long:  `Extract blockchain data and output it in the specified format.`,
 }
 
 func init() {
-	ExtractCmd.PersistentFlags().BoolVarP(&insecure, "insecure", "k", false, "Skip TLS certificate verification (INSECURE)")
-	ExtractCmd.PersistentFlags().BoolVar(&live, "live", false, "Enable live monitoring")
-	ExtractCmd.PersistentFlags().Uint64VarP(&start, "start", "s", 1, "Start block height")
-	ExtractCmd.PersistentFlags().Uint64VarP(&stop, "stop", "e", 1, "Stop block height")
-	ExtractCmd.PersistentFlags().UintVarP(&blockTime, "block-time", "t", 2, "Block time in seconds")
-	ExtractCmd.PersistentFlags().UintVarP(&maxRetries, "max-retries", "r", 3, "Maximum number of retries for failed block processing")
-	ExtractCmd.PersistentFlags().UintVarP(&maxConcurrency, "max-concurrency", "c", 100, "Maximum block retrieval concurrency (advanced)")
+	ExtractCmd.PersistentFlags().BoolP("insecure", "k", false, "Skip TLS certificate verification (INSECURE)")
+	ExtractCmd.PersistentFlags().Bool("live", false, "Enable live monitoring")
+	ExtractCmd.PersistentFlags().Uint64P("start", "s", 1, "Start block height")
+	ExtractCmd.PersistentFlags().Uint64P("stop", "e", 1, "Stop block height")
+	ExtractCmd.PersistentFlags().UintP("block-time", "t", 2, "Block time in seconds")
+	ExtractCmd.PersistentFlags().UintP("max-retries", "r", 3, "Maximum number of retries for failed block processing")
+	ExtractCmd.PersistentFlags().UintP("max-concurrency", "c", 100, "Maximum block retrieval concurrency (advanced)")
+
+	if err := viper.BindPFlags(ExtractCmd.PersistentFlags()); err != nil {
+		slog.Error("Failed to bind ExtractCmd flags", "error", err)
+	}
 
 	ExtractCmd.MarkFlagsMutuallyExclusive("live", "stop")
 
@@ -49,6 +44,16 @@ func init() {
 }
 
 func extract(address string, outputHandler output.OutputHandler) error {
+	insecure := viper.GetBool("insecure")
+	live := viper.GetBool("live")
+	start := viper.GetUint64("start")
+	stop := viper.GetUint64("stop")
+	blockTime := viper.GetUint("block-time")
+	maxConcurrency := viper.GetUint("max-concurrency")
+	maxRetries := viper.GetUint("max-retries")
+
+	slog.Debug("Command-line arguments", "address", address, "insecure", insecure, "live", live, "start", start, "stop", stop, "block_time", blockTime, "max_concurrency", maxConcurrency, "max_retries", maxRetries)
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -61,7 +66,7 @@ func extract(address string, outputHandler output.OutputHandler) error {
 		cancel()
 	}()
 
-	slog.Info("Initializing gRPC client pool", "address", address, "insecure", insecure, "max-concurrency", maxConcurrency, "max-retries", maxRetries)
+	slog.Info("Initializing gRPC client pool...")
 	grpcConn := client.NewGRPCClients(ctx, address, insecure)
 	defer grpcConn.Close()
 

cmd/yaci/json.go

@@ -1,21 +1,24 @@
 package yaci
 
 import (
+	"log/slog"
 	"os"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"github.com/liftedinit/yaci/internal/output"
 )
 
-var jsonOut string
-
 var jsonCmd = &cobra.Command{
 	Use:   "json [address] [flags]",
 	Short: "Extract chain data to JSON files",
 	Args:  cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
+		jsonOut := viper.GetString("json-out")
+		slog.Debug("Command-line argument", "json-out", jsonOut)
+
 		err := os.MkdirAll(jsonOut, 0755)
 		if err != nil {
 			return errors.WithMessage(err, "failed to create output directory")
@@ -32,5 +35,8 @@ var jsonCmd = &cobra.Command{
 }
 
 func init() {
-	jsonCmd.Flags().StringVarP(&jsonOut, "out", "o", "out", "Output directory")
+	jsonCmd.Flags().StringP("json-out", "o", "out", "JSON output directory")
+	if err := viper.BindPFlags(jsonCmd.Flags()); err != nil {
+		slog.Error("Failed to bind jsonCmd flags", "error", err)
+	}
 }

cmd/yaci/postgres.go

@@ -1,25 +1,30 @@
 package yaci
 
 import (
-	"fmt"
+	"log/slog"
 
+	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"github.com/liftedinit/yaci/internal/output"
 )
 
 var postgresCmd = &cobra.Command{
-	Use:   "postgres [address] [psql-connection-string]",
+	Use:   "postgres [address] [flags]",
 	Short: "Extract chain data to a PostgreSQL database",
-	Args:  cobra.ExactArgs(2),
+	Args:  cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
-		connString := args[1]
-		if connString == "" {
-			return fmt.Errorf("connection string is required for PostgreSQL output")
+		postgresConn := viper.GetString("postgres-conn")
+		slog.Debug("Command-line argument", "postgres-conn", postgresConn)
+
+		_, err := pgxpool.ParseConfig(postgresConn)
+		if err != nil {
+			return errors.WithMessage(err, "failed to parse PostgreSQL connection string")
 		}
 
-		outputHandler, err := output.NewPostgresOutputHandler(connString)
+		outputHandler, err := output.NewPostgresOutputHandler(postgresConn)
 		if err != nil {
 			return errors.WithMessage(err, "failed to create PostgreSQL output handler")
 		}
@@ -28,3 +33,10 @@ var postgresCmd = &cobra.Command{
 		return extract(args[0], outputHandler)
 	},
 }
+
+func init() {
+	postgresCmd.Flags().StringP("postgres-conn", "p", "", "PosftgreSQL connection string")
+	if err := viper.BindPFlags(postgresCmd.Flags()); err != nil {
+		slog.Error("Failed to bind postgresCmd flags", "error", err)
+	}
+}

cmd/yaci/root.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 var (
@@ -19,19 +20,18 @@ var (
 		"error": slog.LevelError,
 	}
 	validLogLevelsStr = strings.Join(slices.Sorted(maps.Keys(validLogLevels)), "|")
-	logLevel          string
 )
 
 var rootCmd = &cobra.Command{
 	Use:   "yaci",
 	Short: "Extract chain data",
 	Long:  `yaci connects to a gRPC server and extracts blockchain data.`,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-
+		logLevel := viper.GetString("logLevel")
 		if err := setLogLevel(logLevel); err != nil {
 			return err
 		}
-		slog.Info("Application started", "version", Version)
+		slog.Debug("Application started", "version", Version)
 		return nil
 	},
 }
@@ -52,17 +52,35 @@ func setLogLevel(logLevel string) error {
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVarP(&logLevel, "logLevel", "l", "info", fmt.Sprintf("set log level (%s)", validLogLevelsStr))
+	rootCmd.PersistentFlags().StringP("logLevel", "l", "info", fmt.Sprintf("set log level (%s)", validLogLevelsStr))
+	if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {
+		slog.Error("Failed to bind rootCmd flags", "error", err)
+	}
 
 	rootCmd.SilenceUsage = true
-	rootCmd.SilenceErrors = true // Handled in Execute()
+	rootCmd.SilenceErrors = true
+
+	viper.SetConfigName("config")
+	viper.AddConfigPath(".")
+	viper.AddConfigPath("$HOME/.yaci")
+	viper.AddConfigPath("/etc/yaci")
+
+	viper.SetEnvPrefix("yaci")
+	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	viper.AutomaticEnv()
 
 	rootCmd.AddCommand(ExtractCmd)
 	rootCmd.AddCommand(versionCmd)
 }
 
 // Execute runs the root command.
 func Execute() {
+	if err := viper.ReadInConfig(); err == nil {
+		slog.Info("Using config file", "file", viper.ConfigFileUsed())
+	} else {
+		slog.Info("No config file found")
+	}
+
 	if err := rootCmd.Execute(); err != nil {
 		slog.Error("An error occurred", "error", err)
 		os.Exit(1)

cmd/yaci/tsv.go

@@ -1,21 +1,24 @@
 package yaci
 
 import (
+	"log/slog"
 	"os"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"github.com/liftedinit/yaci/internal/output"
 )
 
-var tsvOut string
-
 var tsvCmd = &cobra.Command{
 	Use:   "tsv [address] [flags]",
 	Short: "Extract chain data to TSV files",
 	Args:  cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
+		tsvOut := viper.GetString("tsv-out")
+		slog.Debug("Command-line argument", "tsv-out", tsvOut)
+
 		err := os.MkdirAll(tsvOut, 0755)
 		if err != nil {
 			return errors.WithMessage(err, "failed to create output directory")
@@ -32,5 +35,8 @@ var tsvCmd = &cobra.Command{
 }
 
 func init() {
-	tsvCmd.Flags().StringVarP(&tsvOut, "out", "o", "tsv", "Output directory")
+	tsvCmd.Flags().StringP("tsv-out", "o", "tsv", "Output directory")
+	if err := viper.BindPFlags(tsvCmd.Flags()); err != nil {
+		slog.Error("Failed to bind tsvCmd flags", "error", err)
+	}
 }

docker-compose.yml

@@ -87,7 +87,7 @@ services:
     build:
       context: ./
       dockerfile: Dockerfile
-    command: ["extract", "postgres", "manifest-ledger:${GRPC}", "postgres://postgres:foobar@db:5432/postgres", "--live", "-k"]
+    command: ["extract", "postgres", "manifest-ledger:${GRPC}", "-p", "postgres://postgres:foobar@db:5432/postgres", "--live", "-k"]
     depends_on:
       db:
         condition: service_healthy

go.mod

@@ -3,15 +3,18 @@ module github.com/liftedinit/yaci
 go 1.23.1
 
 require (
-	github.com/golang/protobuf v1.5.0
+	github.com/golang/protobuf v1.5.3
 	github.com/jackc/pgx/v4 v4.18.3
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.1
+	github.com/spf13/viper v1.19.0
 	google.golang.org/grpc v1.67.1
 	google.golang.org/protobuf v1.35.1
 )
 
 require (
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.14.3 // indirect
@@ -21,10 +24,24 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgtype v1.14.0 // indirect
 	github.com/jackc/puddle v1.3.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/crypto v0.26.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )