Support loading configuration from both YAML files and env vars

go.mod

@@ -2,6 +2,8 @@ module github.com/icinga/icingadb
 
 go 1.22
 
+replace github.com/icinga/icinga-go-library v0.3.1 => github.com/icinga/icinga-go-library v0.3.2-0.20241026175131-ae4d57c2f11f
+
 require (
 	github.com/creasty/defaults v1.8.0
 	github.com/goccy/go-yaml v1.12.0
@@ -23,6 +25,7 @@ require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
+	github.com/caarlos0/env/v11 v11.2.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -33,7 +36,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.12 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/redis/go-redis/v9 v9.5.1 // indirect
+	github.com/redis/go-redis/v9 v9.7.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/ssgreg/journald v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg=
+github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
@@ -32,8 +34,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/icinga/icinga-go-library v0.3.1 h1:PN3cbJJqXQgzRGttuniJPzScbvFhhwmvOJzHweW/HSM=
-github.com/icinga/icinga-go-library v0.3.1/go.mod h1:ZIjlB9ul6B0x71NQR9HuohjZqr8cDS+VRqeBPrdFX6g=
+github.com/icinga/icinga-go-library v0.3.2-0.20241026175131-ae4d57c2f11f h1:KcRkilcsjlMo7oBAjD+aNgCMeiDX4B25ZvRFAWjuN8U=
+github.com/icinga/icinga-go-library v0.3.2-0.20241026175131-ae4d57c2f11f/go.mod h1:mjNyTU1xKlrXYUnKgYNt7gtmFZBbot6uTchFj5HzL0w=
 github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
 github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
@@ -58,8 +60,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
-github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=

internal/command/command.go

@@ -10,6 +10,7 @@ import (
 	icingadbconfig "github.com/icinga/icingadb/internal/config"
 	"github.com/icinga/icingadb/pkg/icingaredis/telemetry"
 	"github.com/pkg/errors"
+	"io/fs"
 	"os"
 	"time"
 )
@@ -37,8 +38,48 @@ func New() *Command {
 		os.Exit(0)
 	}
 
+	// This function supports configuration loading in three scenarios:
+	//
+	// 1. Load configuration solely from YAML files when no relevant environment variables are set.
+	//
+	// 2. Combine YAML file and environment variable configurations, allowing environment variables
+	//    to supplement or override possible incomplete YAML configurations.
+	//
+	// 3. Load entirely from environment variables if the default YAML config file is absent and
+	//    no specific config path is provided by the user.
 	var cfg icingadbconfig.Config
-	if err := config.FromYAMLFile(flags.Config, &cfg); err != nil {
+	var configPath string
+	if flags.Config != "" {
+		configPath = flags.Config
+	} else {
+		configPath = icingadbconfig.DefaultConfigPath
+	}
+
+	if err := config.FromYAMLFile(configPath, &cfg); err != nil {
+		if errors.Is(err, config.ErrInvalidArgument) {
+			panic(err)
+		}
+
+		// Allow continuation with FromEnv by handling:
+		//
+		// - ErrInvalidConfiguration:
+		//   The configuration may be incomplete and will be revalidated in FromEnv.
+		//
+		// - Non-existent file errors:
+		//   If no explicit config path is set, fallback to environment variables is allowed.
+		configIsInvalid := errors.Is(err, config.ErrInvalidConfiguration)
+		defaultConfigFileDoesNotExist := errors.Is(err, fs.ErrNotExist) &&
+			configPath == icingadbconfig.DefaultConfigPath
+		if !(configIsInvalid || defaultConfigFileDoesNotExist) {
+			utils.PrintErrorThenExit(err, 1)
+		}
+	}
+
+	// Call FromEnv regardless of the outcome from FromYAMLFile.
+	// If no environment variables are set, configuration relies entirely on YAML.
+	// Otherwise, environment variables can supplement, override YAML settings, or serve as the sole source.
+	// FromEnv also includes validation, ensuring completeness after considering both sources.
+	if err := config.FromEnv(&cfg, config.EnvOptions{Prefix: "ICINGADB_"}); err != nil {
 		if errors.Is(err, config.ErrInvalidArgument) {
 			panic(err)
 		}

internal/config/config.go

@@ -10,12 +10,15 @@ import (
 	"time"
 )
 
+// DefaultConfigPath specifies the default location of Icinga DB's config.yml for package installations.
+const DefaultConfigPath = "/etc/icingadb/config.yml"
+
 // Config defines Icinga DB config.
 type Config struct {
-	Database  database.Config `yaml:"database"`
-	Redis     redis.Config    `yaml:"redis"`
-	Logging   logging.Config  `yaml:"logging"`
-	Retention RetentionConfig `yaml:"retention"`
+	Database  database.Config `yaml:"database" envPrefix:"DATABASE_"`
+	Redis     redis.Config    `yaml:"redis" envPrefix:"REDIS_"`
+	Logging   logging.Config  `yaml:"logging" envPrefix:"LOGGING_"`
+	Retention RetentionConfig `yaml:"retention" envPrefix:"RETENTION_"`
 }
 
 func (c *Config) SetDefaults() {
@@ -50,7 +53,7 @@ type Flags struct {
 	// Version decides whether to just print the version and exit.
 	Version bool `long:"version" description:"print version and exit"`
 	// Config is the path to the config file
-	Config string `short:"c" long:"config" description:"path to config file" required:"true" default:"/etc/icingadb/config.yml"`
+	Config string `short:"c" long:"config" description:"path to config file"`
 }
 
 // RetentionConfig defines configuration for history retention.