Improve config and logger

.gitignore

@@ -44,7 +44,9 @@ pkg/api/rest/docs/swagger.json
 pkg/api/rest/docs/swagger.yaml
 
 # Config file for CM-Beetle
-conf/
+!conf/
+conf/*
+!conf/template-*
 conf/credentials.conf
 
 # Runtime files

cmd/cm-beetle/main.go

@@ -20,6 +20,11 @@ import (
 	"strconv"
 	"sync"
 
+	// Black import (_) is for running a package's init() function without using its other contents.
+	_ "github.com/cloud-barista/cm-beetle/pkg/config"
+	_ "github.com/cloud-barista/cm-beetle/pkg/logger"
+	"github.com/rs/zerolog/log"
+
 	//_ "github.com/go-sql-driver/mysql"
 	"github.com/fsnotify/fsnotify"
 	_ "github.com/mattn/go-sqlite3"
@@ -28,11 +33,6 @@ import (
 	"github.com/cloud-barista/cm-beetle/pkg/core/common"
 
 	restServer "github.com/cloud-barista/cm-beetle/pkg/api/rest/server"
-
-	// Black import (_) is for running a package's init() function without using its other contents.
-	_ "github.com/cloud-barista/cm-beetle/pkg/config"
-	_ "github.com/cloud-barista/cm-beetle/pkg/logger"
-	"github.com/rs/zerolog/log"
 )
 
 func main() {

conf/config.yaml

@@ -30,6 +30,9 @@ logfile:
 # Set log level, such as trace, debug info, warn, error, fatal, and panic
 loglevel: debug
 
+# Set log writer, such as file, stdout, or both
+logwriter: both
+
 # Set execution environment, such as development or production
 node:
   env: development

conf/setup.env

@@ -18,6 +18,8 @@ export LOGFILE_MAXAGE=30
 export LOGFILE_COMPRESS=false
 # Set log level, such as trace, debug info, warn, error, fatal, and panic
 export LOGLEVEL=debug
+# Set log writer, such as file, stdout, or both
+export LOGWRITER=both
 # Set execution environment, such as development or production
 export NODE_ENV=development
 

conf/template-config.yaml

@@ -0,0 +1,72 @@
+## Set system endpoints
+cmbeetle:
+  root: # To be set in runtime
+
+cbstore:
+  root: # To be set in runtime (based on cmbeetle.root)
+
+cblog:
+  root: # To be set in runtime (based on cmbeetle.root)
+
+cbspider:
+  callmethod: REST
+  rest:
+    url: http://localhost:1024/spider
+
+cbtumblebug:
+  callmethod: REST
+  rest:
+    url: http://localhost:1323/tumblebug
+
+## Logger configuration
+logfile:
+  # Set log file path (default logfile path: ./cm-beetle.log) 
+  path: ./cm-beetle.log
+  maxsize: 10
+  maxbackups: 3
+  maxage: 30
+  compress: false
+
+# Set log level, such as trace, debug info, warn, error, fatal, and panic
+loglevel: debug
+
+# Set log writer, such as file, stdout, or both
+logwriter: both
+
+# Set execution environment, such as development or production
+node:
+  env: development
+
+## Set internal DB config (SQLlite)
+db:
+  url: localhost:3306
+  database: cm_beetle
+  user: cm_beetle
+  password: cm_beetle
+
+## Set API access config
+api:
+  # Set API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all)
+  allow:
+    origins: "*"
+
+  # Set API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path)
+  auth:
+    enabled: true
+
+  username: default
+  password: default
+
+## Set period for auto control goroutine invocation
+autocontrol:
+  duration_ms: 10000
+
+## Set SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056)
+self:
+  endpoint: localhost:8056
+
+## Environment variables that you don't need to touch
+# Swagger UI API document file path 
+apidoc:
+  # export APIDOC_PATH=$CMBEETLE_ROOT/src/api/rest/docs/swagger.json
+  path: # To be set in runtime (based on cmbeetle.root)

conf/template-setup.env

@@ -0,0 +1,48 @@
+## Set system endpoints
+# Set CMBEETLE_ROOT based on path of setup.env relatively
+SCRIPT_DIR=`dirname ${BASH_SOURCE[0]-$0}`
+export CMBEETLE_ROOT=`cd $SCRIPT_DIR && cd .. && pwd`
+export CBSTORE_ROOT=$CMBEETLE_ROOT
+export CBLOG_ROOT=$CMBEETLE_ROOT
+#export CBSPIDER_CALLMETHOD=REST
+#export CBSPIDER_REST_URL=http://localhost:1024/spider
+#export CBTUMBLEBUG_CALLMETHOD=REST
+#export CBTUMBLEBUG_REST_URL=http://localhost:1323/tumblebug
+
+## Logger configuration
+# Set log file path (default logfile path: ./cm-beetle.log) 
+export LOGFILE_PATH=cm-beetle.log
+export LOGFILE_MAXSIZE=10
+export LOGFILE_MAXBACKUPS=3
+export LOGFILE_MAXAGE=30
+export LOGFILE_COMPRESS=false
+# Set log level, such as trace, debug info, warn, error, fatal, and panic
+export LOGLEVEL=debug
+# Set log writer, such as file, stdout, or both
+export LOGWRITER=both
+# Set execution environment, such as development or production
+export NODE_ENV=development
+
+## Set internal DB config (SQLlite)
+export DB_URL=localhost:3306
+export DB_DATABASE=cm_beetle
+export DB_USER=cm_beetlee
+export DB_PASSWORD=cm_beetle
+
+## Set API access config
+# API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all)
+export API_ALLOW_ORIGINS=*
+# Set API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path)
+export API_AUTH_ENABLED=true
+export API_USERNAME=default
+export API_PASSWORD=default
+
+## Set period for auto control goroutine invocation
+export AUTOCONTROL_DURATION_MS=10000
+
+## Set SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056)
+export SELF_ENDPOINT=localhost:8056
+
+## Environment variables that you don't need to touch
+# Swagger UI API document file path 
+export APIDOC_PATH=$CMBEETLE_ROOT/src/api/rest/docs/swagger.json

pkg/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"errors"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
@@ -18,33 +19,55 @@ func Init() {
 	viper.AddConfigPath("../../conf/") // config for development
 	viper.AddConfigPath(".")           // config for production optionally looking for the configuration in the working directory
 	viper.AddConfigPath("./conf/")     // config for production optionally looking for the configuration in the working directory/conf/
-	viper.SetConfigName("config")
 	viper.SetConfigType("yaml")
+	viper.SetConfigName("config")
+
+	// Load main configuration
+	viper.SetConfigName("config")
+	err := viper.ReadInConfig()
+	if err != nil {
+		fmt.Printf("no main config file, using default settings: %s\n", err)
+		log.Printf("no main config file, using default settings: %s", err)
+	}
+
+	// Load secrets configuration
+	// viper.SetConfigName("secrets")
+	// err = viper.MergeInConfig() // Merge in the secrets config
+	// if err != nil {
+	// 	fmt.Printf("no reading secrets config file: %s\n", err)
+	// 	log.Fatalf("no reading secrets config file: %s", err)
+	// }
 
 	// Map environment variable names to config file key names
 	replacer := strings.NewReplacer(".", "_")
 	viper.SetEnvKeyReplacer(replacer)
 
+	// NOTE - the environment variable has higher priority than the config file
 	// Automatically recognize environment variables
 	viper.AutomaticEnv()
 
-	// Try to read the config file in development environment
-	if viper.GetString("node.env") != "production" {
-		if err := viper.ReadInConfig(); err != nil {
-			log.Printf("Error reading config file, using default settings: %s", err)
-		}
-	}
-
 	// Values set in runtime
 	if viper.GetString("cmbeetle.root") == "" {
-		log.Println("cmbeetle.root is not set in config file or environment variable")
-
+		fmt.Println("find project root by using project name")
 		log.Println("find project root by using project name")
+
 		projectName := "cm-beetle"
-		projectRoot, err := findProjectRoot(projectName)
+		// Get the executable path
+		execPath, err := os.Executable()
 		if err != nil {
-			log.Fatalf("Error finding project root directory: %v", err)
+			fmt.Printf("Error getting executable path: %v\n", err)
+			log.Fatalf("Error getting executable path: %v", err)
 		}
+		execDir := filepath.Dir(execPath)
+		projectRoot, err := checkProjectRootInParentDirectory(projectName, execDir)
+		if err != nil {
+			fmt.Printf("set current directory as project root directory (%v)\n", err)
+			log.Printf("set current directory as project root directory (%v)", err)
+			projectRoot = execDir
+		}
+		fmt.Printf("project root directory: %s\n", projectRoot)
+		log.Printf("project root directory: %s\n", projectRoot)
+
 		// Set the binary path
 		viper.Set("cmbeetle.root", projectRoot)
 		viper.Set("cbstore.root", projectRoot)
@@ -54,28 +77,23 @@ func Init() {
 
 	// Recursively print all keys and values in Viper
 	settings := viper.AllSettings()
-	recursivePrintMap(settings, "")
+	if viper.GetString("node.env") == "development" {
+		recursivePrintMap(settings, "")
+	}
 }
 
-func findProjectRoot(projectName string) (string, error) {
-	// Get the executable path
-	execPath, err := os.Executable()
-	if err != nil {
-		log.Fatalf("Error getting executable path: %v", err)
-	}
-	execDir := filepath.Dir(execPath)
+func checkProjectRootInParentDirectory(projectName string, execDir string) (string, error) {
 
-	// find last index of project name
-	index := strings.LastIndex(execDir, projectName)
+	// Append a path separator to the project name for accurate matching
+	projectNameWithSeparator := projectName + string(filepath.Separator)
+	// Find the last index of the project name with the separator
+	index := strings.LastIndex(execDir, projectNameWithSeparator)
 	if index == -1 {
-		log.Println("project name not found the path")
-		return "", errors.New("proejct name not found in the path")
+		return "", errors.New("project name not found in the path")
 	}
 
-	// Cut the string up to the index
-	result := execDir[:index+len(projectName)]
-
-	log.Printf("project root directory: %s\n", result)
+	// Cut the string up to the index + length of the project name
+	result := execDir[:index+len(projectNameWithSeparator)-1]
 
 	return result, nil
 }

pkg/logger/logger.go

@@ -68,31 +68,19 @@ func init() {
 	// Log a message
 	log.Info().
 		Str("logLevel", level.String()).
-		Str("logFilePath", sharedLogFile.Filename).
 		Str("env", env).
 		Int("maxSize", maxSize).
 		Int("maxBackups", maxBackups).
 		Int("maxAge", maxAge).
 		Bool("compress", compress).
 		Msg("Global logger initialized")
-
-	if env == "production" {
-		log.Info().
-			Str("logFilePath", sharedLogFile.Filename).
-			Msg("Single-write setup (logs to file only)")
-	} else {
-		log.Info().
-			Str("logFilePath", sharedLogFile.Filename).
-			Str("ConsoleWriter", "os.Stdout").
-			Msg("Multi-writes setup (logs to both file and console)")
-	}
 }
 
 // Create a new logger
 func NewLogger(level zerolog.Level) *zerolog.Logger {
 
 	// Set config values
-	env := viper.GetString("node.env")
+	logwriter := viper.GetString("logwriter")
 
 	// Multi-writer setup: logs to both file and console
 	multi := zerolog.MultiLevelWriter(
@@ -104,30 +92,40 @@ func NewLogger(level zerolog.Level) *zerolog.Logger {
 
 	// Check the execution environment from the environment variable
 	// Configure the log output
-	if env == "production" {
-		// Apply multi-writer to the global logger
-		logger = zerolog.New(sharedLogFile).Level(level).With().Timestamp().Caller().Logger()
-	} else {
+	if logwriter == "both" {
 		// Apply file to the global logger
 		logger = zerolog.New(multi).Level(level).With().Timestamp().Caller().Logger()
-	}
-
-	// Log a message
-	logger.Info().
-		Str("logLevel", level.String()).
-		Msg("New logger created")
-
-	if env == "production" {
+		logger.Info().
+			Str("logFilePath", sharedLogFile.Filename).
+			Str("ConsoleWriter", "os.Stdout").
+			Msg("Multi-writes setup (logs to both file and console)")
+	} else if logwriter == "file" {
+		// Apply file writer to the global logger
+		logger = zerolog.New(sharedLogFile).Level(level).With().Timestamp().Caller().Logger()
 		logger.Info().
 			Str("logFilePath", sharedLogFile.Filename).
 			Msg("Single-write setup (logs to file only)")
+	} else if logwriter == "stdout" {
+		// Apply ConsoleWriter to the global logger
+		logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stdout}).Level(level).With().Timestamp().Caller().Logger()
+		logger.Info().
+			Str("ConsoleWriter", "os.Stdout").
+			Msg("Single-write setup (logs to console only)")
 	} else {
+		log.Warn().Msgf("Invalid LOGWRITER value: %s. Using default value: both", logwriter)
+		// Apply multi-writer to the global logger
+		logger = zerolog.New(multi).Level(level).With().Timestamp().Caller().Logger()
 		logger.Info().
 			Str("logFilePath", sharedLogFile.Filename).
 			Str("ConsoleWriter", "os.Stdout").
 			Msg("Multi-writes setup (logs to both file and console)")
 	}
 
+	// Log a message
+	logger.Info().
+		Str("logLevel", level.String()).
+		Msg("New logger created")
+
 	return &logger
 }