From 565cfa4bee2a900a37e7d5abdc94bd60f4b0f290 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal Date: Mon, 24 Dec 2018 14:05:27 +0200 Subject: [PATCH] Add flags and docs (#84) --- README.MD | 9 +- .../templates/analyzer-deployment.yaml | 7 +- .../templates/anonymizer-deployment.yaml | 7 +- charts/presidio/templates/api-deployment.yaml | 9 +- .../templates/scheduler-deployment.yaml | 11 ++- charts/presidio/values.yaml | 2 +- docs/install.md | 12 +-- pkg/logger/logger.go | 42 +++++--- pkg/platform/platform.go | 97 +++++++++++++++---- pkg/presidio/item.go | 17 ---- pkg/presidio/presidio.go | 65 +++++++++++++ pkg/presidio/{ => services}/services.go | 57 ++++++++--- pkg/presidio/{ => templates}/templates.go | 34 ++----- pkg/rpc/client.go | 8 +- pkg/rpc/server.go | 4 +- .../cmd/presidio-anonymizer/main.go | 14 +-- presidio-api/cmd/presidio-api/actions.go | 15 ++- presidio-api/cmd/presidio-api/api.go | 13 ++- presidio-api/cmd/presidio-api/main.go | 35 ++++--- presidio-api/cmd/presidio-api/templates.go | 8 +- .../cmd/presidio-collector/main.go | 39 +++++--- .../presidio-collector/processor/processor.go | 4 +- .../cmd/presidio-datasink/main.go | 14 ++- .../cmd/presidio-scheduler/main.go | 59 ++++++----- tests/integration_storage_scanner_test.go | 3 +- 25 files changed, 382 insertions(+), 203 deletions(-) delete mode 100644 pkg/presidio/item.go create mode 100644 pkg/presidio/presidio.go rename pkg/presidio/{ => services}/services.go (63%) rename pkg/presidio/{ => templates}/templates.go (58%) diff --git a/README.MD b/README.MD index f2de8b567..7cbe27ffc 100644 --- a/README.MD +++ b/README.MD @@ -1,4 +1,9 @@ [![Build status](https://dev.azure.com/csedevil/Presidio/_apis/build/status/Presidio-CI)](https://dev.azure.com/csedevil/Presidio/_build/latest?definitionId=48) +[![Go Report Card](https://goreportcard.com/badge/github.com/Microsoft/presidio)](https://goreportcard.com/report/github.com/Microsoft/presidio) +[![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg)](http://opensource.org/licenses/MIT) +![](https://img.shields.io/github/release/Microsoft/presidio.svg) + +--- # Presidio - Data Loss Prevention API @@ -46,7 +51,6 @@ Presidio leverages: * [Redis](https://redis.io/) * [GRPC](https://grpc.io) - The [design document](https://microsoft.github.io/presidio/design.html) introduces Presidio concepts and architecture. ## Quickstart @@ -55,7 +59,6 @@ The [design document](https://microsoft.github.io/presidio/design.html) introduc 2. Create a Presidio project 3. Start using the Presidio analyze and anonymize services - **Note:** Examples are made with [HTTPie](https://httpie.org/) ***Sample 1*** @@ -135,7 +138,7 @@ You can also create reusable templates This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.microsoft.com. +the rights to use your contribution. For details, visit [https://cla.microsoft.com](https://cla.microsoft.com). When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions diff --git a/charts/presidio/templates/analyzer-deployment.yaml b/charts/presidio/templates/analyzer-deployment.yaml index 4d921d825..eb27b5718 100644 --- a/charts/presidio/templates/analyzer-deployment.yaml +++ b/charts/presidio/templates/analyzer-deployment.yaml @@ -1,5 +1,5 @@ {{ $fullname := include "presidio.analyzer.fullname" . }} -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ $fullname }} @@ -10,6 +10,9 @@ metadata: heritage: "{{ .Release.Service }}" spec: replicas: 1 + selector: + matchLabels: + app: {{ $fullname }} template: metadata: labels: @@ -17,7 +20,7 @@ spec: spec: containers: - name: {{ .Chart.Name }} - image: "{{ .Values.registry }}/{{ .Values.analyzer.name }}:{{ default .Chart.Version .Values.analyzer.tag }}" + image: "{{ .Values.registry }}/{{ .Values.analyzer.name }}:{{ default .Chart.AppVersion .Values.analyzer.tag }}" imagePullPolicy: {{ default "IfNotPresent" .Values.analyzer.imagePullPolicy }} ports: - containerPort: {{ .Values.analyzer.service.internalPort }} diff --git a/charts/presidio/templates/anonymizer-deployment.yaml b/charts/presidio/templates/anonymizer-deployment.yaml index c0b01dee4..168a0a578 100644 --- a/charts/presidio/templates/anonymizer-deployment.yaml +++ b/charts/presidio/templates/anonymizer-deployment.yaml @@ -1,5 +1,5 @@ {{ $fullname := include "presidio.anonymizer.fullname" . }} -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ $fullname }} @@ -10,6 +10,9 @@ metadata: heritage: "{{ .Release.Service }}" spec: replicas: 1 + selector: + matchLabels: + app: {{ $fullname }} template: metadata: labels: @@ -17,7 +20,7 @@ spec: spec: containers: - name: {{ .Chart.Name }} - image: "{{ .Values.registry }}/{{ .Values.anonymizer.name }}:{{ default .Chart.Version .Values.anonymizer.tag }}" + image: "{{ .Values.registry }}/{{ .Values.anonymizer.name }}:{{ default .Chart.AppVersion .Values.anonymizer.tag }}" imagePullPolicy: {{ default "IfNotPresent" .Values.anonymizer.imagePullPolicy }} ports: - containerPort: {{ .Values.anonymizer.service.internalPort }} diff --git a/charts/presidio/templates/api-deployment.yaml b/charts/presidio/templates/api-deployment.yaml index 85c829ec7..458fa8115 100644 --- a/charts/presidio/templates/api-deployment.yaml +++ b/charts/presidio/templates/api-deployment.yaml @@ -1,5 +1,5 @@ {{ $fullname := include "presidio.api.fullname" . }} -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ $fullname }} @@ -10,6 +10,9 @@ metadata: heritage: "{{ .Release.Service }}" spec: replicas: 1 + selector: + matchLabels: + app: {{ $fullname }} template: metadata: labels: @@ -18,7 +21,7 @@ spec: serviceAccountName: {{ $fullname }} containers: - name: {{ .Chart.Name }} - image: "{{ .Values.registry }}/{{ .Values.api.name }}:{{ default .Chart.Version .Values.api.tag }}" + image: "{{ .Values.registry }}/{{ .Values.api.name }}:{{ default .Chart.AppVersion .Values.api.tag }}" imagePullPolicy: {{ default "IfNotPresent" .Values.api.imagePullPolicy }} ports: - containerPort: {{ .Values.api.service.internalPort }} @@ -47,5 +50,7 @@ spec: value: {{ template "presidio.anonymizer.address" . }} - name: SCHEDULER_SVC_ADDRESS value: {{ template "presidio.scheduler.address" . }} + - name: GIN_MODE + value: release {{ if .Values.privateRegistry }}imagePullSecrets: - name: {{.Values.privateRegistry}}{{ end }} \ No newline at end of file diff --git a/charts/presidio/templates/scheduler-deployment.yaml b/charts/presidio/templates/scheduler-deployment.yaml index 80ed1a42a..8ad73a8ad 100644 --- a/charts/presidio/templates/scheduler-deployment.yaml +++ b/charts/presidio/templates/scheduler-deployment.yaml @@ -1,6 +1,6 @@ {{ $fullname := include "presidio.scheduler.fullname" . }} -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ $fullname }} @@ -11,6 +11,9 @@ metadata: heritage: "{{ .Release.Service }}" spec: replicas: 1 + selector: + matchLabels: + app: {{ $fullname }} template: metadata: labels: @@ -19,7 +22,7 @@ spec: serviceAccountName: {{ $fullname }} containers: - name: {{ .Chart.Name }} - image: "{{ .Values.registry }}/{{ .Values.scheduler.name }}:{{ default .Chart.Version .Values.scheduler.tag }}" + image: "{{ .Values.registry }}/{{ .Values.scheduler.name }}:{{ default .Chart.AppVersion .Values.scheduler.tag }}" imagePullPolicy: {{ default "IfNotPresent" .Values.scheduler.imagePullPolicy }} ports: - containerPort: {{ .Values.scheduler.service.internalPort }} @@ -39,11 +42,11 @@ spec: - name: DATASINK_GRPC_PORT value: "5000" - name: DATASINK_IMAGE_NAME - value: {{ .Values.registry }}/{{ .Values.datasink.name }}:{{ default .Chart.Version .Values.datasink.tag }} + value: {{ .Values.registry }}/{{ .Values.datasink.name }}:{{ default .Chart.AppVersion .Values.datasink.tag }} - name: DATASINK_PULL_POLICY value: {{ default "IfNotPresent" .Values.datasink.imagePullPolicy }} - name: COLLECTOR_IMAGE_NAME - value: {{ .Values.registry }}/{{ .Values.collector.name }}:{{ default .Chart.Version .Values.collector.tag }} + value: {{ .Values.registry }}/{{ .Values.collector.name }}:{{ default .Chart.AppVersion .Values.collector.tag }} - name: COLLECTOR_IMAGE_PULL_POLICY value: {{ default "IfNotPresent" .Values.collector.imagePullPolicy }} {{ if .Values.privateRegistry }}imagePullSecrets: diff --git a/charts/presidio/values.yaml b/charts/presidio/values.yaml index 824e05a25..8c20ed119 100644 --- a/charts/presidio/values.yaml +++ b/charts/presidio/values.yaml @@ -23,7 +23,7 @@ api: ingress: enabled: true - class: traefik + class: nginx analyzer: name: presidio-analyzer diff --git a/docs/install.md b/docs/install.md index bfb1a1f16..44d1470f2 100644 --- a/docs/install.md +++ b/docs/install.md @@ -40,19 +40,17 @@ $ docker run --rm --name presidio-api --network mynetwork -d -p 8080:8080 -e WEB $ helm install --name redis stable/redis --set usePassword=false,rbac.create=true --namespace presidio-system ``` -3. Install [Traefik](https://github.com/kubernetes/charts/tree/master/stable/traefik) (Optional - Ingress controller for presidio API) +3. Optional - Ingress controller for presidio API. + - [Traefik](https://docs.traefik.io/user-guide/kubernetes/) + - [NGINX](https://docs.microsoft.com/en-us/azure/aks/ingress-tls) - ```sh - $ helm install --name traefik --set rbac.enabled=true stable/traefik --version 1.33.1 --namespace kube-system - ``` - -4. Verify that Redis and Traefik are installed correctly +4. Verify that Redis and Traefik/NGINX are installed correctly 5. Deploy from `/charts/presidio` ```sh # Based on the DOCKER_REGISTRY and PRESIDIO_LABEL from the previous steps - $ helm install --name presidio-demo --set registry=${DOCKER_REGISTRY} . --namespace presidio --version ${PRESIDIO_LABEL} + $ helm install --name presidio-demo --set registry=${DOCKER_REGISTRY},analyzer.tag=${PRESIDIO_LABEL},anonymizer.tag=${PRESIDIO_LABEL},scheduler.tag=${PRESIDIO_LABEL},api.tag=${PRESIDIO_LABEL},collector.tag=${PRESIDIO_LABEL},datasink.tag=${PRESIDIO_LABEL} . --namespace presidio ``` --- diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index c29862c80..aaba72e0c 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -3,7 +3,9 @@ package logger import ( "os" "strings" - "sync" + + "github.com/spf13/pflag" + "github.com/spf13/viper" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -13,18 +15,8 @@ import ( var sugaredLogger *zap.SugaredLogger var logger *zap.Logger -//var logs *observer.ObservedLogs -var once sync.Once - -// Init initializes a thread-safe singleton logger -// This would be called from a main method when the application starts up -// This function would ideally, take zap configuration, but is left out -// in favor of simplicity using the example logger. func init() { - // once ensures the singleton is initialized only once - once.Do(func() { - initLogger() - }) + pflag.String("log_level", "info", "Log level - debug/info/warn/error") } func initLogger() { @@ -49,10 +41,13 @@ func initLogger() { panic(err.Error()) } } + func getLogLevel() zapcore.Level { + logLevel := viper.GetString("LOG_LEVEL") - logLevel := os.Getenv("LOG_LEVEL") switch strings.ToLower(logLevel) { + case "debug": + return zapcore.DebugLevel case "info": return zapcore.InfoLevel case "warn": @@ -66,12 +61,15 @@ func getLogLevel() zapcore.Level { case "fatal": return zapcore.FatalLevel default: - return zapcore.DebugLevel + return zapcore.InfoLevel } } // GetLogger get native not sugared logger func GetLogger() *zap.Logger { + if logger == nil { + initLogger() + } return logger } @@ -85,26 +83,42 @@ func ObserveLogging(level zapcore.Level) *observer.ObservedLogs { // Debug logs a debug message with the given fields func Debug(message string, fields ...interface{}) { + if sugaredLogger == nil { + initLogger() + } sugaredLogger.Debugf(message, fields...) } // Info logs a debug message with the given fields func Info(message string, fields ...interface{}) { + if sugaredLogger == nil { + initLogger() + } sugaredLogger.Infof(message, fields...) } // Warn logs a debug message with the given fields func Warn(message string, fields ...interface{}) { + if sugaredLogger == nil { + initLogger() + } sugaredLogger.Warnf(message, fields...) } // Error logs a debug message with the given fields func Error(message string, fields ...interface{}) { + if sugaredLogger == nil { + initLogger() + } sugaredLogger.Errorf(message, fields...) } // Fatal logs a message than calls os.Exit(1) func Fatal(message string, fields ...interface{}) { + if sugaredLogger == nil { + initLogger() + } sugaredLogger.Fatalf(message, fields...) + sugaredLogger.Sync() os.Exit(1) } diff --git a/pkg/platform/platform.go b/pkg/platform/platform.go index 680d24c9c..025dd844d 100644 --- a/pkg/platform/platform.go +++ b/pkg/platform/platform.go @@ -1,6 +1,8 @@ package platform import ( + "strings" + "github.com/spf13/viper" apiv1 "k8s.io/api/core/v1" @@ -45,14 +47,16 @@ func ConvertPullPolicyStringToType(pullPolicy string) apiv1.PullPolicy { //Settings from all services type Settings struct { - WebPort string - GrpcPort string - DatasinkGrpcPort string + WebPort int + GrpcPort int + DatasinkGrpcPort int Namespace string AnalyzerSvcAddress string AnonymizerSvcAddress string SchedulerSvcAddress string RedisURL string + RedisPassword string + RedisDB int DatasinkImage string CollectorImage string DatasinkImagePullPolicy string @@ -62,28 +66,85 @@ type Settings struct { QueueURL string } +//WebPort for http server +const WebPort = "web_port" + +//GrpcPort for GRPC server +const GrpcPort = "grpc_port" + +//DatasinkGrpcPort for data sink GRPC server +const DatasinkGrpcPort = "datasink_grpc_port" + +//PresidioNamespace for k8s deployment +const PresidioNamespace = "presidio_namespace" + +//AnalyzerSvcAddress analyzer service address +const AnalyzerSvcAddress = "analyzer_svc_address" + +//AnonymizerSvcAddress anonymizer service address +const AnonymizerSvcAddress = "anonymizer_svc_address" + +//SchedulerSvcAddress scheduler service address +const SchedulerSvcAddress = "scheduler_svc_address" + +//RedisURL redis address +const RedisURL = "redis_url" + +//RedisDb redis db number +const RedisDb = "redis_db" + +//RedisPassword redis db password +const RedisPassword = "redis_password" + +//DatasinkImageName datasink docker image name +const DatasinkImageName = "datasink_image_name" + +//DatasinkImagePullPolicy datasink image k8s pull policy +const DatasinkImagePullPolicy = "datasink_image_pull_policy" + +//CollectorImageName collector docker image name +const CollectorImageName = "collector_image_name" + +//CollectorImagePullPolicy collector image k8s pull policy +const CollectorImagePullPolicy = "collector_image_pull_policy" + +//ScannerRequest template +const ScannerRequest = "scanner_request" + +//StreamRequest template +const StreamRequest = "stream_request" + +//QueueURL rabbitmq url +const QueueURL = "queue_url" + //GetSettings from env vars func GetSettings() *Settings { viper.AutomaticEnv() settings := Settings{ - WebPort: viper.GetString("WEB_PORT"), - GrpcPort: viper.GetString("GRPC_PORT"), - DatasinkGrpcPort: viper.GetString("DATASINK_GRPC_PORT"), - Namespace: viper.GetString("PRESIDIO_NAMESPACE"), - AnalyzerSvcAddress: viper.GetString("ANALYZER_SVC_ADDRESS"), - AnonymizerSvcAddress: viper.GetString("ANONYMIZER_SVC_ADDRESS"), - SchedulerSvcAddress: viper.GetString("SCHEDULER_SVC_ADDRESS"), - RedisURL: viper.GetString("REDIS_URL"), - DatasinkImage: viper.GetString("DATASINK_IMAGE_NAME"), - CollectorImage: viper.GetString("COLLECTOR_IMAGE_NAME"), - DatasinkImagePullPolicy: viper.GetString("DATASINK_IMAGE_PULL_POLICY"), - CollectorImagePullPolicy: viper.GetString("COLLECTOR_IMAGE_PULL_POLICY"), - ScannerRequest: viper.GetString("SCANNER_REQUEST"), - StreamRequest: viper.GetString("STREAM_REQUEST"), - QueueURL: viper.GetString("QUEUE_URL"), + WebPort: viper.GetInt(strings.ToUpper(WebPort)), + GrpcPort: viper.GetInt(strings.ToUpper(GrpcPort)), + DatasinkGrpcPort: viper.GetInt(strings.ToUpper(DatasinkGrpcPort)), + Namespace: getTrimmedEnv(PresidioNamespace), + AnalyzerSvcAddress: getTrimmedEnv(AnalyzerSvcAddress), + AnonymizerSvcAddress: getTrimmedEnv(AnonymizerSvcAddress), + SchedulerSvcAddress: getTrimmedEnv(SchedulerSvcAddress), + RedisURL: getTrimmedEnv(RedisURL), + RedisDB: viper.GetInt(strings.ToUpper(RedisDb)), + RedisPassword: getTrimmedEnv(RedisPassword), + DatasinkImage: getTrimmedEnv(DatasinkImageName), + CollectorImage: getTrimmedEnv(CollectorImageName), + DatasinkImagePullPolicy: getTrimmedEnv(DatasinkImagePullPolicy), + CollectorImagePullPolicy: getTrimmedEnv(CollectorImagePullPolicy), + ScannerRequest: getTrimmedEnv(ScannerRequest), + StreamRequest: getTrimmedEnv(StreamRequest), + QueueURL: getTrimmedEnv(QueueURL), } return &settings } + +func getTrimmedEnv(key string) string { + return strings.Trim(viper.GetString(strings.ToUpper(key)), " ") +} diff --git a/pkg/presidio/item.go b/pkg/presidio/item.go deleted file mode 100644 index 98c234f45..000000000 --- a/pkg/presidio/item.go +++ /dev/null @@ -1,17 +0,0 @@ -package presidio - -// Item interface represent the supported item's methods. -type Item interface { - - //GetUniqueID returns the scanned item unique id - GetUniqueID() (string, error) - - //GetContent returns the content of the scanned item - GetContent() (string, error) - - //GetPath returns the item path - GetPath() string - - //IsContentTypeSupported returns if the item can be scanned. - IsContentTypeSupported() error -} diff --git a/pkg/presidio/presidio.go b/pkg/presidio/presidio.go new file mode 100644 index 000000000..e624b2d18 --- /dev/null +++ b/pkg/presidio/presidio.go @@ -0,0 +1,65 @@ +package presidio + +import ( + "context" + "encoding/json" + + types "github.com/Microsoft/presidio-genproto/golang" + "github.com/Microsoft/presidio/pkg/cache" +) + +//ServicesAPI interface for services action +type ServicesAPI interface { + SetupAnalyzerService() + SetupAnonymizerService() + SetupSchedulerService() + SetupDatasinkService() + SetupCache() cache.Cache + AnalyzeItem(ctx context.Context, text string, template *types.AnalyzeTemplate) ([]*types.AnalyzeResult, error) + AnonymizeItem(ctx context.Context, analyzeResults []*types.AnalyzeResult, text string, anonymizeTemplate *types.AnonymizeTemplate) (*types.AnonymizeResponse, error) + SendResultToDatasink(ctx context.Context, analyzeResults []*types.AnalyzeResult, + anonymizeResults *types.AnonymizeResponse, path string) error + ApplyStream(ctx context.Context, streamsJobRequest *types.StreamsJobRequest) (*types.StreamsJobResponse, error) + ApplyScan(ctx context.Context, scanJobRequest *types.ScannerCronJobRequest) (*types.ScannerCronJobResponse, error) + InitDatasink(ctx context.Context, datasinkTemplate *types.DatasinkTemplate) (*types.DatasinkResponse, error) + CloseDatasink(ctx context.Context, datasinkTemplate *types.CompletionMessage) (*types.DatasinkResponse, error) +} + +//TemplatesStore interface for template actions +type TemplatesStore interface { + GetTemplate(project string, action string, id string) (string, error) + InsertTemplate(project string, action string, id string, value string) error + UpdateTemplate(project string, action string, id string, value string) error + DeleteTemplate(project string, action string, id string) error +} + +// Item interface represent the supported item's methods. +type Item interface { + + //GetUniqueID returns the scanned item unique id + GetUniqueID() (string, error) + + //GetContent returns the content of the scanned item + GetContent() (string, error) + + //GetPath returns the item path + GetPath() string + + //IsContentTypeSupported returns if the item can be scanned. + IsContentTypeSupported() error +} + +// ConvertJSONToInterface convert Json to go Interface +func ConvertJSONToInterface(template string, convertTo interface{}) error { + err := json.Unmarshal([]byte(template), &convertTo) + return err +} + +// ConvertInterfaceToJSON convert go interface to json +func ConvertInterfaceToJSON(template interface{}) (string, error) { + b, err := json.Marshal(template) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/pkg/presidio/services.go b/pkg/presidio/services/services.go similarity index 63% rename from pkg/presidio/services.go rename to pkg/presidio/services/services.go index 5e4e9171e..cfc1c51a4 100644 --- a/pkg/presidio/services.go +++ b/pkg/presidio/services/services.go @@ -1,4 +1,4 @@ -package presidio +package services import ( "fmt" @@ -11,26 +11,32 @@ import ( "github.com/Microsoft/presidio/pkg/cache/redis" log "github.com/Microsoft/presidio/pkg/logger" "github.com/Microsoft/presidio/pkg/platform" + "github.com/Microsoft/presidio/pkg/presidio" "github.com/Microsoft/presidio/pkg/rpc" ) -var settings *platform.Settings = platform.GetSettings() - //Services exposes GRPC services type Services struct { AnalyzerService types.AnalyzeServiceClient AnonymizeService types.AnonymizeServiceClient DatasinkService types.DatasinkServiceClient SchedulerService types.SchedulerServiceClient + Settings *platform.Settings +} + +//New services with settings +func New(settings *platform.Settings) presidio.ServicesAPI { + svc := Services{Settings: settings} + return &svc } //SetupAnalyzerService GRPC connection func (services *Services) SetupAnalyzerService() { - if settings.AnalyzerSvcAddress == "" { + if services.Settings.AnalyzerSvcAddress == "" { log.Fatal("analyzer service address is empty") } - analyzeService, err := rpc.SetupAnalyzerService(settings.AnalyzerSvcAddress) + analyzeService, err := rpc.SetupAnalyzerService(services.Settings.AnalyzerSvcAddress) if err != nil { log.Fatal("Connection to analyzer service failed %q", err) } @@ -41,11 +47,11 @@ func (services *Services) SetupAnalyzerService() { //SetupAnonymizerService GRPC connection func (services *Services) SetupAnonymizerService() { - if settings.AnonymizerSvcAddress == "" { + if services.Settings.AnonymizerSvcAddress == "" { log.Fatal("anonymizer service address is empty") } - anonymizeService, err := rpc.SetupAnonymizeService(settings.AnonymizerSvcAddress) + anonymizeService, err := rpc.SetupAnonymizeService(services.Settings.AnonymizerSvcAddress) if err != nil { log.Fatal("Connection to anonymizer service failed %q", err) } @@ -57,12 +63,12 @@ func (services *Services) SetupAnonymizerService() { //SetupSchedulerService GRPC connection func (services *Services) SetupSchedulerService() { - if settings.SchedulerSvcAddress == "" { + if services.Settings.SchedulerSvcAddress == "" { log.Warn("scheduler service address is empty") return } - schedulerService, err := rpc.SetupSchedulerService(settings.SchedulerSvcAddress) + schedulerService, err := rpc.SetupSchedulerService(services.Settings.SchedulerSvcAddress) if err != nil { log.Fatal("Connection to anonymizer service failed %q", err) } @@ -73,7 +79,7 @@ func (services *Services) SetupSchedulerService() { //SetupDatasinkService GRPC connection func (services *Services) SetupDatasinkService() { address := "localhost" - datasinkService, err := rpc.SetupDatasinkService(fmt.Sprintf("%s:%s", address, settings.DatasinkGrpcPort)) + datasinkService, err := rpc.SetupDatasinkService(fmt.Sprintf("%s:%d", address, services.Settings.DatasinkGrpcPort)) if err != nil { log.Fatal("Connection to datasink service failed %q", err) } @@ -82,16 +88,15 @@ func (services *Services) SetupDatasinkService() { } //SetupCache Redis cache -func SetupCache() cache.Cache { - if settings.RedisURL == "" { +func (services *Services) SetupCache() cache.Cache { + if services.Settings.RedisURL == "" { log.Fatal("redis address is empty") } - // TODO: change the password and DB defaults cache := redis.New( - settings.RedisURL, - "", // no password set - 0, // use default DB + services.Settings.RedisURL, + services.Settings.RedisPassword, + services.Settings.RedisDB, ) return cache } @@ -144,3 +149,23 @@ func (services *Services) SendResultToDatasink(ctx context.Context, analyzeResul _, err := services.DatasinkService.Apply(ctx, datasinkRequest) return err } + +//ApplyStream create stream job +func (services *Services) ApplyStream(ctx context.Context, streamsJobRequest *types.StreamsJobRequest) (*types.StreamsJobResponse, error) { + return services.SchedulerService.ApplyStream(ctx, streamsJobRequest) +} + +//ApplyScan create scan cron job +func (services *Services) ApplyScan(ctx context.Context, scanJobRequest *types.ScannerCronJobRequest) (*types.ScannerCronJobResponse, error) { + return services.SchedulerService.ApplyScan(ctx, scanJobRequest) +} + +//InitDatasink initialize datasink app +func (services *Services) InitDatasink(ctx context.Context, datasinkTemplate *types.DatasinkTemplate) (*types.DatasinkResponse, error) { + return services.DatasinkService.Init(ctx, datasinkTemplate) +} + +//CloseDatasink notify datasink the collector is finished +func (services *Services) CloseDatasink(ctx context.Context, datasinkTemplate *types.CompletionMessage) (*types.DatasinkResponse, error) { + return services.DatasinkService.Completion(ctx, datasinkTemplate) +} diff --git a/pkg/presidio/templates.go b/pkg/presidio/templates/templates.go similarity index 58% rename from pkg/presidio/templates.go rename to pkg/presidio/templates/templates.go index f487d2e7d..a80654162 100644 --- a/pkg/presidio/templates.go +++ b/pkg/presidio/templates/templates.go @@ -1,10 +1,10 @@ -package presidio +package templates import ( - "encoding/json" "fmt" "github.com/Microsoft/presidio/pkg/platform" + "github.com/Microsoft/presidio/pkg/presidio" ) const separator = "." @@ -15,30 +15,31 @@ type Templates struct { } //New KV store -func New(s platform.Store) *Templates { +func New(s platform.Store) presidio.TemplatesStore { return &Templates{platformStore: s} } -// CreateKey creates template key in the structure: project/action/id -func CreateKey(project string, action string, id string) string { +// createKey creates template key in the structure: project/action/id +func createKey(project string, action string, id string) string { key := fmt.Sprintf("%s%s%s%s%s", project, separator, action, separator, id) return key } // GetTemplate from key store -func (templates *Templates) GetTemplate(key string) (string, error) { +func (templates *Templates) GetTemplate(project string, action string, id string) (string, error) { + key := createKey(project, action, id) return templates.platformStore.GetKVPair(key) } // InsertTemplate inserts a template to the key store func (templates *Templates) InsertTemplate(project string, action string, id string, value string) error { - key := CreateKey(project, action, id) + key := createKey(project, action, id) return templates.platformStore.PutKVPair(key, value) } // UpdateTemplate updates the template in the key store func (templates *Templates) UpdateTemplate(project string, action string, id string, value string) error { - key := CreateKey(project, action, id) + key := createKey(project, action, id) err := templates.platformStore.DeleteKVPair(key) if err != nil { return err @@ -48,21 +49,6 @@ func (templates *Templates) UpdateTemplate(project string, action string, id str // DeleteTemplate deletes a template from key store func (templates *Templates) DeleteTemplate(project string, action string, id string) error { - key := CreateKey(project, action, id) + key := createKey(project, action, id) return templates.platformStore.DeleteKVPair(key) } - -// ConvertJSONToInterface convert Json to go Interface -func ConvertJSONToInterface(template string, convertTo interface{}) error { - err := json.Unmarshal([]byte(template), &convertTo) - return err -} - -// ConvertInterfaceToJSON convert go interface to json -func ConvertInterfaceToJSON(template interface{}) (string, error) { - b, err := json.Marshal(template) - if err != nil { - return "", err - } - return string(b), nil -} diff --git a/pkg/rpc/client.go b/pkg/rpc/client.go index ef3ca6135..e2d2712ef 100644 --- a/pkg/rpc/client.go +++ b/pkg/rpc/client.go @@ -37,7 +37,7 @@ func connect(addr string) (*grpc.ClientConn, error) { return conn, nil } -//SetupAnonymizeService ... +//SetupAnonymizeService connect to anonymizer service with GRPC func SetupAnonymizeService(address string) (types.AnonymizeServiceClient, error) { conn, err := connect(address) @@ -49,7 +49,7 @@ func SetupAnonymizeService(address string) (types.AnonymizeServiceClient, error) return client, nil } -//SetupAnalyzerService ... +//SetupAnalyzerService connect to analyzer service with GRPC func SetupAnalyzerService(address string) (types.AnalyzeServiceClient, error) { conn, err := connect(address) if err != nil { @@ -59,7 +59,7 @@ func SetupAnalyzerService(address string) (types.AnalyzeServiceClient, error) { return client, nil } -//SetupDatasinkService ... +//SetupDatasinkService connect to datasink service with GRPC func SetupDatasinkService(address string) (types.DatasinkServiceClient, error) { conn, err := connect(address) if err != nil { @@ -69,7 +69,7 @@ func SetupDatasinkService(address string) (types.DatasinkServiceClient, error) { return client, nil } -//SetupSchedulerService ... +//SetupSchedulerService connect to scheduler service with GRPC func SetupSchedulerService(address string) (types.SchedulerServiceClient, error) { conn, err := connect(address) if err != nil { diff --git a/pkg/rpc/server.go b/pkg/rpc/server.go index 6da359d84..74e271034 100644 --- a/pkg/rpc/server.go +++ b/pkg/rpc/server.go @@ -10,9 +10,9 @@ import ( ) //SetupClient setup grpc listener -func SetupClient(grpcPort string) (net.Listener, *grpc.Server) { +func SetupClient(grpcPort int) (net.Listener, *grpc.Server) { - addr := fmt.Sprintf(":%s", grpcPort) + addr := fmt.Sprintf(":%d", grpcPort) log.Info(addr) lis, err := net.Listen("tcp", addr) if err != nil { diff --git a/presidio-anonymizer/cmd/presidio-anonymizer/main.go b/presidio-anonymizer/cmd/presidio-anonymizer/main.go index 5da128eb5..39a34bc8b 100644 --- a/presidio-anonymizer/cmd/presidio-anonymizer/main.go +++ b/presidio-anonymizer/cmd/presidio-anonymizer/main.go @@ -6,10 +6,12 @@ import ( types "github.com/Microsoft/presidio-genproto/golang" + "github.com/spf13/pflag" + "github.com/spf13/viper" + log "github.com/Microsoft/presidio/pkg/logger" "github.com/Microsoft/presidio/pkg/platform" "github.com/Microsoft/presidio/pkg/rpc" - "github.com/Microsoft/presidio/presidio-anonymizer/cmd/presidio-anonymizer/anonymizer" ) @@ -17,17 +19,17 @@ type server struct{} func main() { - settings := platform.GetSettings() + pflag.Int(platform.GrpcPort, 3001, "GRPC listen port") - if settings.GrpcPort == "" { - log.Fatal("GRPC_PORT (currently [%s]) env var must me set.", settings.GrpcPort) - } + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + + settings := platform.GetSettings() lis, s := rpc.SetupClient(settings.GrpcPort) types.RegisterAnonymizeServiceServer(s, &server{}) reflection.Register(s) - if err := s.Serve(lis); err != nil { log.Fatal(err.Error()) } diff --git a/presidio-api/cmd/presidio-api/actions.go b/presidio-api/cmd/presidio-api/actions.go index a19edcfd2..1e8534847 100644 --- a/presidio-api/cmd/presidio-api/actions.go +++ b/presidio-api/cmd/presidio-api/actions.go @@ -13,11 +13,9 @@ import ( ) func (api *API) setupGRPCServices() { - svc := presidio.Services{} - svc.SetupAnalyzerService() - svc.SetupAnonymizerService() - svc.SetupSchedulerService() - api.Services = &svc + api.Services.SetupAnalyzerService() + api.Services.SetupAnonymizerService() + api.Services.SetupSchedulerService() } func (api *API) analyze(c *gin.Context) { @@ -95,7 +93,7 @@ func (api *API) scheduleScannerCronJob(c *gin.Context) { func (api *API) invokeScannerCronJobScheduler(scannerCronJobRequest *types.ScannerCronJobRequest, c *gin.Context) *types.ScannerCronJobResponse { - res, err := api.Services.SchedulerService.ApplyScan(c, scannerCronJobRequest) + res, err := api.Services.ApplyScan(c, scannerCronJobRequest) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return nil @@ -166,7 +164,7 @@ func (api *API) scheduleStreamsJob(c *gin.Context) { } func (api *API) invokeStreamsJobScheduler(streamsJobRequest *types.StreamsJobRequest, c *gin.Context) *types.StreamsJobResponse { - res, err := api.Services.SchedulerService.ApplyStream(c, streamsJobRequest) + res, err := api.Services.ApplyStream(c, streamsJobRequest) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return nil @@ -215,8 +213,7 @@ func (api *API) getStreamsJobRequest(jobAPIRequest *types.StreamsJobApiRequest, } func (api *API) getTemplate(project string, action string, id string, obj interface{}, c *gin.Context) { - key := presidio.CreateKey(project, action, id) - template, err := api.Templates.GetTemplate(key) + template, err := api.Templates.GetTemplate(project, action, id) if err != nil { c.AbortWithError(http.StatusBadRequest, err) } diff --git a/presidio-api/cmd/presidio-api/api.go b/presidio-api/cmd/presidio-api/api.go index 75934e4a5..0e894cc0b 100644 --- a/presidio-api/cmd/presidio-api/api.go +++ b/presidio-api/cmd/presidio-api/api.go @@ -3,6 +3,8 @@ package main import ( "github.com/Microsoft/presidio/pkg/platform" "github.com/Microsoft/presidio/pkg/presidio" + "github.com/Microsoft/presidio/pkg/presidio/services" + "github.com/Microsoft/presidio/pkg/presidio/templates" ) const ( @@ -17,15 +19,16 @@ const ( //API kv store type API struct { - Templates *presidio.Templates - Services *presidio.Services + Templates presidio.TemplatesStore + Services presidio.ServicesAPI } //New KV store -func New(s platform.Store) *API { - template := presidio.New(s) +func New(store platform.Store, settings *platform.Settings) *API { + template := templates.New(store) + svc := services.New(settings) return &API{ Templates: template, - Services: &presidio.Services{}, + Services: svc, } } diff --git a/presidio-api/cmd/presidio-api/main.go b/presidio-api/cmd/presidio-api/main.go index 1e4c563a9..9516990f5 100644 --- a/presidio-api/cmd/presidio-api/main.go +++ b/presidio-api/cmd/presidio-api/main.go @@ -2,7 +2,9 @@ package main import ( "os" - "strconv" + + "github.com/spf13/pflag" + "github.com/spf13/viper" log "github.com/Microsoft/presidio/pkg/logger" "github.com/Microsoft/presidio/pkg/platform" @@ -13,18 +15,19 @@ import ( func main() { - settings := platform.GetSettings() - if settings.WebPort == "" { - log.Fatal("WEB_PORT env var must me set.") - } - - port, err := strconv.Atoi(settings.WebPort) - if err != nil { - log.Fatal(err.Error()) - } + pflag.Int(platform.WebPort, 8080, "HTTP listen port") + pflag.String(platform.AnalyzerSvcAddress, "localhost:3000", "Analyzer service address") + pflag.String(platform.AnonymizerSvcAddress, "localhost:3001", "Anonymizer service address") + pflag.String(platform.SchedulerSvcAddress, "", "Scheduler service address") + pflag.String(platform.RedisURL, "", "Redis address") + pflag.String(platform.RedisPassword, "", "Redis db password (optional)") + pflag.Int(platform.RedisDb, 0, "Redis db (optional)") + pflag.String(platform.PresidioNamespace, "", "Presidio Kubernetes namespace (optional)") - r := server.Setup(port) + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + settings := platform.GetSettings() var api *API // Kubernetes platform @@ -33,17 +36,23 @@ func main() { if err != nil { log.Fatal(err.Error()) } - api = New(store) + api = New(store, settings) } else { // Local platform store, err := local.New(os.TempDir()) if err != nil { log.Fatal(err.Error()) } - api = New(store) + api = New(store, settings) } api.setupGRPCServices() + setupHTTPServer(api, settings.WebPort) +} + +func setupHTTPServer(api *API, port int) { + + r := server.Setup(port) // api/v1 group v1 := r.Group("/api/v1") diff --git a/presidio-api/cmd/presidio-api/templates.go b/presidio-api/cmd/presidio-api/templates.go index 68a8c64d2..b5fc37cf5 100644 --- a/presidio-api/cmd/presidio-api/templates.go +++ b/presidio-api/cmd/presidio-api/templates.go @@ -7,8 +7,7 @@ import ( "github.com/gin-gonic/gin" types "github.com/Microsoft/presidio-genproto/golang" - - pkg_templates "github.com/Microsoft/presidio/pkg/presidio" + "github.com/Microsoft/presidio/pkg/presidio" server "github.com/Microsoft/presidio/pkg/server" ) @@ -24,8 +23,7 @@ func (api *API) getActionTemplate(c *gin.Context) { action := c.Param("action") project := c.Param("project") id := c.Param("id") - key := pkg_templates.CreateKey(project, action, id) - result, err := api.Templates.GetTemplate(key) + result, err := api.Templates.GetTemplate(project, action, id) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -107,7 +105,7 @@ func validateTemplate(action string, c *gin.Context) (string, error) { func bindAndConvert(template interface{}, c *gin.Context) (string, error) { if c.BindJSON(&template) == nil { - return pkg_templates.ConvertInterfaceToJSON(template) + return presidio.ConvertInterfaceToJSON(template) } return "", fmt.Errorf("No template found") } diff --git a/presidio-collector/cmd/presidio-collector/main.go b/presidio-collector/cmd/presidio-collector/main.go index 41ded2972..29580ec44 100644 --- a/presidio-collector/cmd/presidio-collector/main.go +++ b/presidio-collector/cmd/presidio-collector/main.go @@ -5,9 +5,13 @@ import ( types "github.com/Microsoft/presidio-genproto/golang" + "github.com/spf13/pflag" + "github.com/spf13/viper" + log "github.com/Microsoft/presidio/pkg/logger" "github.com/Microsoft/presidio/pkg/platform" "github.com/Microsoft/presidio/pkg/presidio" + "github.com/Microsoft/presidio/pkg/presidio/services" "github.com/Microsoft/presidio/presidio-collector/cmd/presidio-collector/processor" "github.com/Microsoft/presidio/presidio-collector/cmd/presidio-collector/scanner" "github.com/Microsoft/presidio/presidio-collector/cmd/presidio-collector/streams" @@ -18,22 +22,32 @@ var scanRequest *types.ScanRequest func main() { + pflag.String(platform.AnalyzerSvcAddress, "localhost:3000", "Analyzer service address") + pflag.String(platform.AnonymizerSvcAddress, "localhost:3001", "Anonymizer service address") + pflag.String(platform.RedisURL, "localhost:6379", "Redis address") + pflag.String(platform.RedisPassword, "", "Redis db password (optional)") + pflag.Int(platform.RedisDb, 0, "Redis db") + pflag.String(platform.ScannerRequest, "", "Scanner request") + pflag.String(platform.StreamRequest, "", "Stream request") + + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + settings := platform.GetSettings() parseRequest(settings) - svc := presidio.Services{} - + svc := services.New(settings) svc.SetupAnalyzerService() if streamRequest.StreamConfig != nil { st := streams.CreateStream(context.Background(), streamRequest) - setupDatasinkService(&svc, streamRequest.DatasinkTemplate) + setupDatasinkService(svc, streamRequest.DatasinkTemplate) if streamRequest.AnonymizeTemplate != nil { svc.SetupAnonymizerService() } - err := processor.ReceiveEventsFromStream(st, &svc, streamRequest) + err := processor.ReceiveEventsFromStream(st, svc, streamRequest) if err != nil { log.Error(err.Error()) } @@ -41,8 +55,8 @@ func main() { } if scanRequest.ScanTemplate != nil { - cache := presidio.SetupCache() - setupDatasinkService(&svc, scanRequest.DatasinkTemplate) + cache := svc.SetupCache() + setupDatasinkService(svc, scanRequest.DatasinkTemplate) if scanRequest.AnonymizeTemplate != nil { svc.SetupAnonymizerService() } @@ -50,33 +64,28 @@ func main() { // Scan ctx := context.Background() - err := processor.ScanStorage(ctx, scan, cache, &svc, scanRequest) + err := processor.ScanStorage(ctx, scan, cache, svc, scanRequest) if err != nil { log.Fatal(err.Error()) } // notify datasink that scanner is done - - svc.DatasinkService.Completion(ctx, &types.CompletionMessage{}) + svc.CloseDatasink(ctx, &types.CompletionMessage{}) log.Info("Done!") } } -func setupDatasinkService(svc *presidio.Services, datasinkTemplate *types.DatasinkTemplate) { +func setupDatasinkService(svc presidio.ServicesAPI, datasinkTemplate *types.DatasinkTemplate) { svc.SetupDatasinkService() - _, err := svc.DatasinkService.Init(context.Background(), datasinkTemplate) + _, err := svc.InitDatasink(context.Background(), datasinkTemplate) if err != nil { log.Fatal(err.Error()) } } func parseRequest(settings *platform.Settings) { - if settings.GrpcPort == "" { - // Set to default - settings.GrpcPort = "5000" - } streamRequest = &types.StreamRequest{} presidio.ConvertJSONToInterface(settings.StreamRequest, streamRequest) diff --git a/presidio-collector/cmd/presidio-collector/processor/processor.go b/presidio-collector/cmd/presidio-collector/processor/processor.go index 2355007b6..4da71c128 100644 --- a/presidio-collector/cmd/presidio-collector/processor/processor.go +++ b/presidio-collector/cmd/presidio-collector/processor/processor.go @@ -15,7 +15,7 @@ import ( ) //ReceiveEventsFromStream .. -func ReceiveEventsFromStream(st stream.Stream, services *presidio.Services, streamRequest *types.StreamRequest) error { +func ReceiveEventsFromStream(st stream.Stream, services presidio.ServicesAPI, streamRequest *types.StreamRequest) error { return st.Receive(func(ctx context.Context, partition string, sequence string, text string) error { analyzerResult, err := services.AnalyzeItem(ctx, text, streamRequest.AnalyzeTemplate) @@ -45,7 +45,7 @@ func ReceiveEventsFromStream(st stream.Stream, services *presidio.Services, stre } //ScanStorage .. -func ScanStorage(ctx context.Context, scan scanner.Scanner, cache cache.Cache, services *presidio.Services, scanRequest *types.ScanRequest) error { +func ScanStorage(ctx context.Context, scan scanner.Scanner, cache cache.Cache, services presidio.ServicesAPI, scanRequest *types.ScanRequest) error { return scan.Scan(func(item interface{}) error { var analyzerResult []*types.AnalyzeResult diff --git a/presidio-datasink/cmd/presidio-datasink/main.go b/presidio-datasink/cmd/presidio-datasink/main.go index 4bf644ca6..dec9a53b9 100644 --- a/presidio-datasink/cmd/presidio-datasink/main.go +++ b/presidio-datasink/cmd/presidio-datasink/main.go @@ -7,6 +7,9 @@ import ( "os" "strings" + "github.com/spf13/pflag" + "github.com/spf13/viper" + log "github.com/Microsoft/presidio/pkg/logger" "google.golang.org/grpc" @@ -29,12 +32,13 @@ var ( type server struct{} func main() { - settings := platform.GetSettings() - if settings.DatasinkGrpcPort == "" { - // Set to default - settings.DatasinkGrpcPort = "5000" - } + pflag.Int(platform.DatasinkGrpcPort, 5000, "GRPC listen port") + + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + + settings := platform.GetSettings() // Setup server lis, grpcServer = rpc.SetupClient(settings.DatasinkGrpcPort) diff --git a/presidio-scheduler/cmd/presidio-scheduler/main.go b/presidio-scheduler/cmd/presidio-scheduler/main.go index 55fa10e50..102c7226e 100644 --- a/presidio-scheduler/cmd/presidio-scheduler/main.go +++ b/presidio-scheduler/cmd/presidio-scheduler/main.go @@ -8,8 +8,12 @@ import ( "google.golang.org/grpc/reflection" apiv1 "k8s.io/api/core/v1" - types "github.com/Microsoft/presidio-genproto/golang" + "strings" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + types "github.com/Microsoft/presidio-genproto/golang" log "github.com/Microsoft/presidio/pkg/logger" "github.com/Microsoft/presidio/pkg/platform" "github.com/Microsoft/presidio/pkg/platform/kube" @@ -24,21 +28,20 @@ var ( ) func main() { - settings = platform.GetSettings() - if settings.GrpcPort == "" { - log.Fatal("GRPC_PORT (currently [%s]) env var must me set.", settings.GrpcPort) - } - if settings.AnalyzerSvcAddress == "" { - log.Fatal("analyzer service address is empty") - } - if settings.AnonymizerSvcAddress == "" { - log.Fatal("anonymizer service address is empty") - } + pflag.Int(platform.GrpcPort, 3002, "GRPC listen port") + pflag.Int(platform.DatasinkGrpcPort, 5000, "Datasink GRPC listen port") + pflag.String(platform.AnalyzerSvcAddress, "localhost:3000", "Analyzer service address") + pflag.String(platform.AnonymizerSvcAddress, "localhost:3001", "Anonymizer service address") + pflag.String(platform.RedisURL, "localhost:6379", "Redis address") + pflag.String(platform.RedisPassword, "", "Redis db password (optional)") + pflag.Int(platform.RedisDb, 0, "Redis db (optional)") + pflag.String(platform.PresidioNamespace, "", "Presidio Kubernetes namespace (optional)") - if settings.RedisURL == "" { - log.Fatal("redis service address is empty") - } + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + + settings = platform.GetSettings() var err error log.Info("namespace %s", settings.Namespace) @@ -89,7 +92,7 @@ func applyScanRequest(r *types.ScannerCronJobRequest) (*types.ScannerCronJobResp Name: "datasink", Image: settings.DatasinkImage, EnvVars: []apiv1.EnvVar{ - {Name: "DATASINK_GRPC_PORT", Value: settings.DatasinkGrpcPort}, + {Name: strings.ToUpper(platform.DatasinkGrpcPort), Value: fmt.Sprintf("%d", settings.DatasinkGrpcPort)}, }, ImagePullPolicy: datasinkPolicy, }, @@ -97,11 +100,13 @@ func applyScanRequest(r *types.ScannerCronJobRequest) (*types.ScannerCronJobResp Name: "scanner", Image: settings.CollectorImage, EnvVars: []apiv1.EnvVar{ - {Name: "DATASINK_GRPC_PORT", Value: settings.DatasinkGrpcPort}, - {Name: "REDIS_URL", Value: settings.RedisURL}, - {Name: "ANALYZER_SVC_ADDRESS", Value: settings.AnalyzerSvcAddress}, - {Name: "ANONYMIZER_SVC_ADDRESS", Value: settings.AnonymizerSvcAddress}, - {Name: "SCANNER_REQUEST", Value: string(scanRequest)}, + {Name: strings.ToUpper(platform.DatasinkGrpcPort), Value: fmt.Sprintf("%d", settings.DatasinkGrpcPort)}, + {Name: strings.ToUpper(platform.RedisURL), Value: settings.RedisURL}, + {Name: strings.ToUpper(platform.RedisPassword), Value: settings.RedisPassword}, + {Name: strings.ToUpper(platform.RedisDb), Value: fmt.Sprintf("%d", settings.RedisDB)}, + {Name: strings.ToUpper(platform.AnalyzerSvcAddress), Value: settings.AnalyzerSvcAddress}, + {Name: strings.ToUpper(platform.AnonymizerSvcAddress), Value: settings.AnonymizerSvcAddress}, + {Name: strings.ToUpper(platform.ScannerRequest), Value: string(scanRequest)}, }, ImagePullPolicy: collectorPolicy, }, @@ -134,7 +139,7 @@ func applyStreamRequest(r *types.StreamsJobRequest) (*types.StreamsJobResponse, Name: "datasink", Image: settings.DatasinkImage, EnvVars: []apiv1.EnvVar{ - {Name: "DATASINK_GRPC_PORT", Value: settings.DatasinkGrpcPort}, + {Name: strings.ToUpper(platform.DatasinkGrpcPort), Value: fmt.Sprintf("%d", settings.DatasinkGrpcPort)}, }, ImagePullPolicy: datasinkPolicy, }, @@ -142,11 +147,13 @@ func applyStreamRequest(r *types.StreamsJobRequest) (*types.StreamsJobResponse, Name: "streams", Image: settings.CollectorImage, EnvVars: []apiv1.EnvVar{ - {Name: "DATASINK_GRPC_PORT", Value: settings.DatasinkGrpcPort}, - {Name: "REDIS_URL", Value: settings.RedisURL}, - {Name: "ANALYZER_SVC_ADDRESS", Value: settings.AnalyzerSvcAddress}, - {Name: "ANONYMIZER_SVC_ADDRESS", Value: settings.AnonymizerSvcAddress}, - {Name: "STREAM_REQUEST", Value: string(streamRequest)}, + {Name: strings.ToUpper(platform.DatasinkGrpcPort), Value: fmt.Sprintf("%d", settings.DatasinkGrpcPort)}, + {Name: strings.ToUpper(platform.RedisURL), Value: settings.RedisURL}, + {Name: strings.ToUpper(platform.RedisPassword), Value: settings.RedisPassword}, + {Name: strings.ToUpper(platform.RedisDb), Value: fmt.Sprintf("%d", settings.RedisDB)}, + {Name: strings.ToUpper(platform.AnalyzerSvcAddress), Value: settings.AnalyzerSvcAddress}, + {Name: strings.ToUpper(platform.AnonymizerSvcAddress), Value: settings.AnonymizerSvcAddress}, + {Name: strings.ToUpper(platform.StreamRequest), Value: string(streamRequest)}, }, ImagePullPolicy: collectorPolicy, }, diff --git a/tests/integration_storage_scanner_test.go b/tests/integration_storage_scanner_test.go index 640e95607..be76dad4b 100644 --- a/tests/integration_storage_scanner_test.go +++ b/tests/integration_storage_scanner_test.go @@ -20,6 +20,7 @@ import ( cache_mock "github.com/Microsoft/presidio/pkg/cache/mock" log "github.com/Microsoft/presidio/pkg/logger" "github.com/Microsoft/presidio/pkg/presidio" + "github.com/Microsoft/presidio/pkg/presidio/services" "github.com/Microsoft/presidio/pkg/storage" "github.com/Microsoft/presidio/presidio-collector/cmd/presidio-collector/processor" "github.com/Microsoft/presidio/presidio-collector/cmd/presidio-collector/scanner" @@ -41,7 +42,7 @@ var ( s3Config stow.ConfigMap containerName = "test" testCache c.Cache - grpcSvc = presidio.Services{} + grpcSvc = services.Services{} ) func TestMain(m *testing.M) {