From 1c9aec9fbb7cb6c15c83a44075892e141ac440bf Mon Sep 17 00:00:00 2001 From: Yunkon Kim Date: Thu, 18 Jan 2024 20:41:47 +0900 Subject: [PATCH] Improve config and logger * Relocate the blank import to resolve init() order issue between pkgs * Add `logwriter` option and update logger * Improve config for use regardless of node environment * Add templates of config.yaml and setup.env --- .gitignore | 4 ++- cmd/cm-beetle/main.go | 10 +++--- conf/config.yaml | 3 ++ conf/setup.env | 2 ++ conf/template-config.yaml | 72 +++++++++++++++++++++++++++++++++++++ conf/template-setup.env | 48 +++++++++++++++++++++++++ pkg/config/config.go | 74 ++++++++++++++++++++++++--------------- pkg/logger/logger.go | 48 ++++++++++++------------- 8 files changed, 202 insertions(+), 59 deletions(-) create mode 100644 conf/template-config.yaml create mode 100644 conf/template-setup.env diff --git a/.gitignore b/.gitignore index fe60439..eb6973e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,7 +44,9 @@ pkg/api/rest/docs/swagger.json pkg/api/rest/docs/swagger.yaml # Config file for CM-Beetle -conf/ +!conf/ +conf/* +!conf/template-* conf/credentials.conf # Runtime files diff --git a/cmd/cm-beetle/main.go b/cmd/cm-beetle/main.go index 9a21954..c3b8371 100644 --- a/cmd/cm-beetle/main.go +++ b/cmd/cm-beetle/main.go @@ -20,6 +20,11 @@ import ( "strconv" "sync" + // Black import (_) is for running a package's init() function without using its other contents. + _ "github.com/cloud-barista/cm-beetle/pkg/config" + _ "github.com/cloud-barista/cm-beetle/pkg/logger" + "github.com/rs/zerolog/log" + //_ "github.com/go-sql-driver/mysql" "github.com/fsnotify/fsnotify" _ "github.com/mattn/go-sqlite3" @@ -28,11 +33,6 @@ import ( "github.com/cloud-barista/cm-beetle/pkg/core/common" restServer "github.com/cloud-barista/cm-beetle/pkg/api/rest/server" - - // Black import (_) is for running a package's init() function without using its other contents. - _ "github.com/cloud-barista/cm-beetle/pkg/config" - _ "github.com/cloud-barista/cm-beetle/pkg/logger" - "github.com/rs/zerolog/log" ) func main() { diff --git a/conf/config.yaml b/conf/config.yaml index aa97c92..9ecd3b2 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -30,6 +30,9 @@ logfile: # Set log level, such as trace, debug info, warn, error, fatal, and panic loglevel: debug +# Set log writer, such as file, stdout, or both +logwriter: both + # Set execution environment, such as development or production node: env: development diff --git a/conf/setup.env b/conf/setup.env index efff2c4..07aca0d 100644 --- a/conf/setup.env +++ b/conf/setup.env @@ -18,6 +18,8 @@ export LOGFILE_MAXAGE=30 export LOGFILE_COMPRESS=false # Set log level, such as trace, debug info, warn, error, fatal, and panic export LOGLEVEL=debug +# Set log writer, such as file, stdout, or both +export LOGWRITER=both # Set execution environment, such as development or production export NODE_ENV=development diff --git a/conf/template-config.yaml b/conf/template-config.yaml new file mode 100644 index 0000000..9ecd3b2 --- /dev/null +++ b/conf/template-config.yaml @@ -0,0 +1,72 @@ +## Set system endpoints +cmbeetle: + root: # To be set in runtime + +cbstore: + root: # To be set in runtime (based on cmbeetle.root) + +cblog: + root: # To be set in runtime (based on cmbeetle.root) + +cbspider: + callmethod: REST + rest: + url: http://localhost:1024/spider + +cbtumblebug: + callmethod: REST + rest: + url: http://localhost:1323/tumblebug + +## Logger configuration +logfile: + # Set log file path (default logfile path: ./cm-beetle.log) + path: ./cm-beetle.log + maxsize: 10 + maxbackups: 3 + maxage: 30 + compress: false + +# Set log level, such as trace, debug info, warn, error, fatal, and panic +loglevel: debug + +# Set log writer, such as file, stdout, or both +logwriter: both + +# Set execution environment, such as development or production +node: + env: development + +## Set internal DB config (SQLlite) +db: + url: localhost:3306 + database: cm_beetle + user: cm_beetle + password: cm_beetle + +## Set API access config +api: + # Set API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all) + allow: + origins: "*" + + # Set API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path) + auth: + enabled: true + + username: default + password: default + +## Set period for auto control goroutine invocation +autocontrol: + duration_ms: 10000 + +## Set SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056) +self: + endpoint: localhost:8056 + +## Environment variables that you don't need to touch +# Swagger UI API document file path +apidoc: + # export APIDOC_PATH=$CMBEETLE_ROOT/src/api/rest/docs/swagger.json + path: # To be set in runtime (based on cmbeetle.root) diff --git a/conf/template-setup.env b/conf/template-setup.env new file mode 100644 index 0000000..07aca0d --- /dev/null +++ b/conf/template-setup.env @@ -0,0 +1,48 @@ +## Set system endpoints +# Set CMBEETLE_ROOT based on path of setup.env relatively +SCRIPT_DIR=`dirname ${BASH_SOURCE[0]-$0}` +export CMBEETLE_ROOT=`cd $SCRIPT_DIR && cd .. && pwd` +export CBSTORE_ROOT=$CMBEETLE_ROOT +export CBLOG_ROOT=$CMBEETLE_ROOT +#export CBSPIDER_CALLMETHOD=REST +#export CBSPIDER_REST_URL=http://localhost:1024/spider +#export CBTUMBLEBUG_CALLMETHOD=REST +#export CBTUMBLEBUG_REST_URL=http://localhost:1323/tumblebug + +## Logger configuration +# Set log file path (default logfile path: ./cm-beetle.log) +export LOGFILE_PATH=cm-beetle.log +export LOGFILE_MAXSIZE=10 +export LOGFILE_MAXBACKUPS=3 +export LOGFILE_MAXAGE=30 +export LOGFILE_COMPRESS=false +# Set log level, such as trace, debug info, warn, error, fatal, and panic +export LOGLEVEL=debug +# Set log writer, such as file, stdout, or both +export LOGWRITER=both +# Set execution environment, such as development or production +export NODE_ENV=development + +## Set internal DB config (SQLlite) +export DB_URL=localhost:3306 +export DB_DATABASE=cm_beetle +export DB_USER=cm_beetlee +export DB_PASSWORD=cm_beetle + +## Set API access config +# API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all) +export API_ALLOW_ORIGINS=* +# Set API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path) +export API_AUTH_ENABLED=true +export API_USERNAME=default +export API_PASSWORD=default + +## Set period for auto control goroutine invocation +export AUTOCONTROL_DURATION_MS=10000 + +## Set SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056) +export SELF_ENDPOINT=localhost:8056 + +## Environment variables that you don't need to touch +# Swagger UI API document file path +export APIDOC_PATH=$CMBEETLE_ROOT/src/api/rest/docs/swagger.json \ No newline at end of file diff --git a/pkg/config/config.go b/pkg/config/config.go index 59e4f61..87947b7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "errors" + "fmt" "log" "os" "path/filepath" @@ -18,33 +19,55 @@ func Init() { viper.AddConfigPath("../../conf/") // config for development viper.AddConfigPath(".") // config for production optionally looking for the configuration in the working directory viper.AddConfigPath("./conf/") // config for production optionally looking for the configuration in the working directory/conf/ - viper.SetConfigName("config") viper.SetConfigType("yaml") + viper.SetConfigName("config") + + // Load main configuration + viper.SetConfigName("config") + err := viper.ReadInConfig() + if err != nil { + fmt.Printf("no main config file, using default settings: %s\n", err) + log.Printf("no main config file, using default settings: %s", err) + } + + // Load secrets configuration + // viper.SetConfigName("secrets") + // err = viper.MergeInConfig() // Merge in the secrets config + // if err != nil { + // fmt.Printf("no reading secrets config file: %s\n", err) + // log.Fatalf("no reading secrets config file: %s", err) + // } // Map environment variable names to config file key names replacer := strings.NewReplacer(".", "_") viper.SetEnvKeyReplacer(replacer) + // NOTE - the environment variable has higher priority than the config file // Automatically recognize environment variables viper.AutomaticEnv() - // Try to read the config file in development environment - if viper.GetString("node.env") != "production" { - if err := viper.ReadInConfig(); err != nil { - log.Printf("Error reading config file, using default settings: %s", err) - } - } - // Values set in runtime if viper.GetString("cmbeetle.root") == "" { - log.Println("cmbeetle.root is not set in config file or environment variable") - + fmt.Println("find project root by using project name") log.Println("find project root by using project name") + projectName := "cm-beetle" - projectRoot, err := findProjectRoot(projectName) + // Get the executable path + execPath, err := os.Executable() if err != nil { - log.Fatalf("Error finding project root directory: %v", err) + fmt.Printf("Error getting executable path: %v\n", err) + log.Fatalf("Error getting executable path: %v", err) } + execDir := filepath.Dir(execPath) + projectRoot, err := checkProjectRootInParentDirectory(projectName, execDir) + if err != nil { + fmt.Printf("set current directory as project root directory (%v)\n", err) + log.Printf("set current directory as project root directory (%v)", err) + projectRoot = execDir + } + fmt.Printf("project root directory: %s\n", projectRoot) + log.Printf("project root directory: %s\n", projectRoot) + // Set the binary path viper.Set("cmbeetle.root", projectRoot) viper.Set("cbstore.root", projectRoot) @@ -54,28 +77,23 @@ func Init() { // Recursively print all keys and values in Viper settings := viper.AllSettings() - recursivePrintMap(settings, "") + if viper.GetString("node.env") == "development" { + recursivePrintMap(settings, "") + } } -func findProjectRoot(projectName string) (string, error) { - // Get the executable path - execPath, err := os.Executable() - if err != nil { - log.Fatalf("Error getting executable path: %v", err) - } - execDir := filepath.Dir(execPath) +func checkProjectRootInParentDirectory(projectName string, execDir string) (string, error) { - // find last index of project name - index := strings.LastIndex(execDir, projectName) + // Append a path separator to the project name for accurate matching + projectNameWithSeparator := projectName + string(filepath.Separator) + // Find the last index of the project name with the separator + index := strings.LastIndex(execDir, projectNameWithSeparator) if index == -1 { - log.Println("project name not found the path") - return "", errors.New("proejct name not found in the path") + return "", errors.New("project name not found in the path") } - // Cut the string up to the index - result := execDir[:index+len(projectName)] - - log.Printf("project root directory: %s\n", result) + // Cut the string up to the index + length of the project name + result := execDir[:index+len(projectNameWithSeparator)-1] return result, nil } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 394c1b5..93542d5 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -68,31 +68,19 @@ func init() { // Log a message log.Info(). Str("logLevel", level.String()). - Str("logFilePath", sharedLogFile.Filename). Str("env", env). Int("maxSize", maxSize). Int("maxBackups", maxBackups). Int("maxAge", maxAge). Bool("compress", compress). Msg("Global logger initialized") - - if env == "production" { - log.Info(). - Str("logFilePath", sharedLogFile.Filename). - Msg("Single-write setup (logs to file only)") - } else { - log.Info(). - Str("logFilePath", sharedLogFile.Filename). - Str("ConsoleWriter", "os.Stdout"). - Msg("Multi-writes setup (logs to both file and console)") - } } // Create a new logger func NewLogger(level zerolog.Level) *zerolog.Logger { // Set config values - env := viper.GetString("node.env") + logwriter := viper.GetString("logwriter") // Multi-writer setup: logs to both file and console multi := zerolog.MultiLevelWriter( @@ -104,30 +92,40 @@ func NewLogger(level zerolog.Level) *zerolog.Logger { // Check the execution environment from the environment variable // Configure the log output - if env == "production" { - // Apply multi-writer to the global logger - logger = zerolog.New(sharedLogFile).Level(level).With().Timestamp().Caller().Logger() - } else { + if logwriter == "both" { // Apply file to the global logger logger = zerolog.New(multi).Level(level).With().Timestamp().Caller().Logger() - } - - // Log a message - logger.Info(). - Str("logLevel", level.String()). - Msg("New logger created") - - if env == "production" { + logger.Info(). + Str("logFilePath", sharedLogFile.Filename). + Str("ConsoleWriter", "os.Stdout"). + Msg("Multi-writes setup (logs to both file and console)") + } else if logwriter == "file" { + // Apply file writer to the global logger + logger = zerolog.New(sharedLogFile).Level(level).With().Timestamp().Caller().Logger() logger.Info(). Str("logFilePath", sharedLogFile.Filename). Msg("Single-write setup (logs to file only)") + } else if logwriter == "stdout" { + // Apply ConsoleWriter to the global logger + logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stdout}).Level(level).With().Timestamp().Caller().Logger() + logger.Info(). + Str("ConsoleWriter", "os.Stdout"). + Msg("Single-write setup (logs to console only)") } else { + log.Warn().Msgf("Invalid LOGWRITER value: %s. Using default value: both", logwriter) + // Apply multi-writer to the global logger + logger = zerolog.New(multi).Level(level).With().Timestamp().Caller().Logger() logger.Info(). Str("logFilePath", sharedLogFile.Filename). Str("ConsoleWriter", "os.Stdout"). Msg("Multi-writes setup (logs to both file and console)") } + // Log a message + logger.Info(). + Str("logLevel", level.String()). + Msg("New logger created") + return &logger }