Create redis cache for templates (#85)

charts/presidio/templates/api-deployment.yaml

@@ -44,13 +44,17 @@ spec:
             value: {{ .Values.api.service.internalPort | quote }}
           - name: REDIS_URL
             value: {{ .Values.redis.url }}
+          - name: REDIS_PASSWORD
+            value: {{ .Values.redis.password | default "" }}
+          - name: REDIS_DB
+            value: {{ .Values.redis.db | default "0" | quote}}
+          - name: REDIS_SSL
+            value: {{ .Values.redis.ssl | default "false" | quote}}
           - name: ANALYZER_SVC_ADDRESS
             value:  {{ template "presidio.analyzer.address" . }}
           - name: ANONYMIZER_SVC_ADDRESS
             value:  {{ template "presidio.anonymizer.address" . }}
           - name: SCHEDULER_SVC_ADDRESS
             value:  {{ template "presidio.scheduler.address" . }}
-          - name: GIN_MODE
-            value: release
       {{ if .Values.privateRegistry }}imagePullSecrets:
         - name: {{.Values.privateRegistry}}{{ end }}

charts/presidio/templates/scheduler-deployment.yaml

@@ -34,7 +34,13 @@ spec:
           - name: GRPC_PORT
             value: {{ .Values.scheduler.service.internalPort | quote }}
           - name: REDIS_URL
-            value: {{ .Values.redis.url}}
+            value: {{ .Values.redis.url }}
+          - name: REDIS_PASSWORD
+            value: {{ .Values.redis.password | default "" }}
+          - name: REDIS_DB
+            value: {{ .Values.redis.db | default "0" | quote}}
+          - name: REDIS_SSL
+            value: {{ .Values.redis.ssl | default "false" | quote}}
           - name: ANALYZER_SVC_ADDRESS
             value:  {{ template "presidio.analyzer.address" . }}
           - name: ANONYMIZER_SVC_ADDRESS

charts/presidio/values.yaml

@@ -5,6 +5,9 @@ registry: presidio.azurecr.io
 
 redis:
     url: redis-master.presidio-system.svc.cluster.local:6379
+    #password:
+    #db: 0
+    #ssl: false
 
 api:
   name: presidio-api

pkg/cache/cache.go

@@ -4,4 +4,5 @@ package cache
 type Cache interface {
 	Set(key string, value string) error
 	Get(key string) (string, error)
+	Delete(key string) error
 }

pkg/cache/mock/cache.go

@@ -1,7 +1,10 @@
 package mock
 
 import (
+	"fmt"
+
 	"github.com/Microsoft/presidio/pkg/cache"
+	log "github.com/Microsoft/presidio/pkg/logger"
 )
 
 type mockCache struct {
@@ -19,15 +22,29 @@ func New() cache.Cache {
 func (c *mockCache) Set(key string, value string) error {
 	cache := *c
 	cache.c[key] = value
+	log.Debug("Mock - Set key %s", key)
 	return nil
 }
 
 //Get key value
 func (c *mockCache) Get(key string) (string, error) {
 	cache := *c
 	if val, ok := cache.c[key]; ok {
+		log.Debug("Mock - Get key %s", key)
 		return val, nil
 	}
 
 	return "", nil
 }
+
+//Delete key value
+func (c *mockCache) Delete(key string) error {
+	cache := *c
+	if _, ok := cache.c[key]; ok {
+		log.Debug("Mock - Delete key %s", key)
+		delete(cache.c, key)
+		return nil
+	}
+
+	return fmt.Errorf("Key doesn't exist")
+}

pkg/cache/redis/redis.go

@@ -1,34 +1,50 @@
 package redis
 
 import (
+	"crypto/tls"
+
 	r "github.com/go-redis/redis"
 
 	"github.com/Microsoft/presidio/pkg/cache"
+	log "github.com/Microsoft/presidio/pkg/logger"
 )
 
 type redis struct {
 	client *r.Client
 }
 
 //New Return new Redis cache
-func New(address string, password string, db int) cache.Cache {
+func New(address string, password string, db int, ssl bool) cache.Cache {
+
+	if ssl {
+		return &redis{
+			client: r.NewClient(&r.Options{
+				Addr:      address,
+				Password:  password,
+				DB:        db,
+				TLSConfig: &tls.Config{InsecureSkipVerify: true},
+			})}
+	}
 	return &redis{
 		client: r.NewClient(&r.Options{
 			Addr:     address,
 			Password: password,
 			DB:       db,
-		}),
-	}
+		})}
+
 }
 
 //Set key value
 func (c *redis) Set(key string, value string) error {
-	return c.client.Set(key, value, 0).Err()
+	err := c.client.Set(key, value, 0).Err()
+	log.Debug("Redis - Set key %s", key)
+	return err
 }
 
 //Get key value
 func (c *redis) Get(key string) (string, error) {
 	val, err := c.client.Get(key).Result()
+	log.Debug("Redis - Get key %s", key)
 	if err == r.Nil {
 		return "", nil
 	} else if err != nil {
@@ -37,3 +53,10 @@ func (c *redis) Get(key string) (string, error) {
 		return val, nil
 	}
 }
+
+//Delete key
+func (c *redis) Delete(key string) error {
+	err := c.client.Del(key).Err()
+	log.Debug("Redis - Delete key %s", key)
+	return err
+}

pkg/logger/logger.go

@@ -3,48 +3,48 @@ package logger
 import (
 	"os"
 	"strings"
-
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
+	"sync"
 
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zaptest/observer"
 )
 
-var sugaredLogger *zap.SugaredLogger
-var logger *zap.Logger
+var once sync.Once
 
 func init() {
-	pflag.String("log_level", "info", "Log level - debug/info/warn/error")
+	// once ensures the singleton is initialized only once
+	once.Do(func() {
+		CreateLogger("debug")
+	})
 }
 
-func initLogger() {
-	logLevel := getLogLevel()
-	level := zap.NewAtomicLevelAt(logLevel)
+//CreateLogger with specific log level
+func CreateLogger(logLevel string) {
+
+	level := getLogLevel(logLevel)
+	alevel := zap.NewAtomicLevelAt(level)
 
 	var config zap.Config
-	if logLevel == zapcore.DebugLevel {
+	if level == zapcore.DebugLevel {
 		config = zap.NewDevelopmentConfig()
 	} else {
 		config = zap.NewProductionConfig()
 	}
 	config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
 	config.DisableStacktrace = true
 	config.DisableCaller = true
-	config.Level = level
+	config.Level = alevel
+
+	logger, err := config.Build()
+	zap.ReplaceGlobals(logger)
 
-	build, err := config.Build()
-	logger = build
-	sugaredLogger = build.Sugar()
 	if err != nil {
 		panic(err.Error())
 	}
 }
 
-func getLogLevel() zapcore.Level {
-	logLevel := viper.GetString("LOG_LEVEL")
-
+func getLogLevel(logLevel string) zapcore.Level {
 	switch strings.ToLower(logLevel) {
 	case "debug":
 		return zapcore.DebugLevel
@@ -67,58 +67,41 @@ func getLogLevel() zapcore.Level {
 
 // GetLogger get native not sugared logger
 func GetLogger() *zap.Logger {
-	if logger == nil {
-		initLogger()
-	}
-	return logger
+	return zap.L()
 }
 
 // ObserveLogging constructs a logger through the zap/zaptest/observer framework
 // so that logs will be accessible in tests.
 func ObserveLogging(level zapcore.Level) *observer.ObservedLogs {
 	observedLogger, logs := observer.New(level)
-	sugaredLogger = zap.New(observedLogger).With().Sugar()
+	logger := zap.New(observedLogger)
+	zap.ReplaceGlobals(logger)
 	return logs
 }
 
 // Debug logs a debug message with the given fields
 func Debug(message string, fields ...interface{}) {
-	if sugaredLogger == nil {
-		initLogger()
-	}
-	sugaredLogger.Debugf(message, fields...)
+	zap.S().Debugf(message, fields...)
 }
 
 // Info logs a debug message with the given fields
 func Info(message string, fields ...interface{}) {
-	if sugaredLogger == nil {
-		initLogger()
-	}
-	sugaredLogger.Infof(message, fields...)
+	zap.S().Infof(message, fields...)
 }
 
 // Warn logs a debug message with the given fields
 func Warn(message string, fields ...interface{}) {
-	if sugaredLogger == nil {
-		initLogger()
-	}
-	sugaredLogger.Warnf(message, fields...)
+	zap.S().Warnf(message, fields...)
 }
 
 // Error logs a debug message with the given fields
 func Error(message string, fields ...interface{}) {
-	if sugaredLogger == nil {
-		initLogger()
-	}
-	sugaredLogger.Errorf(message, fields...)
+	zap.S().Errorf(message, fields...)
 }
 
 // Fatal logs a message than calls os.Exit(1)
 func Fatal(message string, fields ...interface{}) {
-	if sugaredLogger == nil {
-		initLogger()
-	}
-	sugaredLogger.Fatalf(message, fields...)
-	sugaredLogger.Sync()
+	zap.S().Fatalf(message, fields...)
+	zap.S().Sync()
 	os.Exit(1)
 }

pkg/platform/platform.go

@@ -57,13 +57,15 @@ type Settings struct {
 	RedisURL                 string
 	RedisPassword            string
 	RedisDB                  int
+	RedisSSL                 bool
 	DatasinkImage            string
 	CollectorImage           string
 	DatasinkImagePullPolicy  string
 	CollectorImagePullPolicy string
 	ScannerRequest           string
 	StreamRequest            string
 	QueueURL                 string
+	LogLevel                 string
 }
 
 //WebPort for http server
@@ -93,6 +95,9 @@ const RedisURL = "redis_url"
 //RedisDb redis db number
 const RedisDb = "redis_db"
 
+//RedisSSL redis ssl
+const RedisSSL = "redis_ssl"
+
 //RedisPassword redis db password
 const RedisPassword = "redis_password"
 
@@ -117,6 +122,9 @@ const StreamRequest = "stream_request"
 //QueueURL rabbitmq url
 const QueueURL = "queue_url"
 
+//LogLevel debug/info/warn/error/fatal
+const LogLevel = "log_level"
+
 //GetSettings from env vars
 func GetSettings() *Settings {
 
@@ -132,6 +140,7 @@ func GetSettings() *Settings {
 		SchedulerSvcAddress:      getTrimmedEnv(SchedulerSvcAddress),
 		RedisURL:                 getTrimmedEnv(RedisURL),
 		RedisDB:                  viper.GetInt(strings.ToUpper(RedisDb)),
+		RedisSSL:                 viper.GetBool(strings.ToUpper(RedisSSL)),
 		RedisPassword:            getTrimmedEnv(RedisPassword),
 		DatasinkImage:            getTrimmedEnv(DatasinkImageName),
 		CollectorImage:           getTrimmedEnv(CollectorImageName),
@@ -140,6 +149,7 @@ func GetSettings() *Settings {
 		ScannerRequest:           getTrimmedEnv(ScannerRequest),
 		StreamRequest:            getTrimmedEnv(StreamRequest),
 		QueueURL:                 getTrimmedEnv(QueueURL),
+		LogLevel:                 getTrimmedEnv(LogLevel),
 	}
 
 	return &settings

pkg/presidio/services/services.go

@@ -97,6 +97,7 @@ func (services *Services) SetupCache() cache.Cache {
 		services.Settings.RedisURL,
 		services.Settings.RedisPassword,
 		services.Settings.RedisDB,
+		services.Settings.RedisSSL,
 	)
 	return cache
 }