From 633699bb12919785c6d3743707ecd12731203652 Mon Sep 17 00:00:00 2001 From: Ildar Nurislamov Date: Tue, 9 Aug 2022 11:21:49 +0300 Subject: [PATCH 1/2] Allow redis database selection for all Server and Configurator redis connection pools (not in Redis Source yet) --- configurator/backend/main.go | 8 ++++++-- documentation/configuration/index.mdx | 1 + documentation/deployment/scale.mdx | 5 +++++ server/drivers/redis/redis.go | 4 ++-- server/main.go | 12 ++---------- server/meta/redis_factory.go | 7 ++++++- server/meta/redis_factory_test.go | 10 +++++----- server/meta/storage.go | 4 +++- server/users/storage.go | 4 +++- 9 files changed, 33 insertions(+), 22 deletions(-) diff --git a/configurator/backend/main.go b/configurator/backend/main.go index 3838aed32..0536720ad 100644 --- a/configurator/backend/main.go +++ b/configurator/backend/main.go @@ -343,8 +343,10 @@ func newAuthorizator(ctx context.Context, vp *viper.Viper, mailSender authorizat port := vp.GetInt("auth.redis.port") sentinelMaster := vp.GetString("auth.redis.sentinel_master_name") redisPassword := vp.GetString("auth.redis.password") + redisDatabase := vp.GetInt("auth.redis.database") + tlsSkipVerify := vp.GetBool("auth.redis.tls_skip_verify") - redisPoolFactory := meta.NewRedisPoolFactory(host, port, redisPassword, tlsSkipVerify, sentinelMaster) + redisPoolFactory := meta.NewRedisPoolFactory(host, port, redisPassword, redisDatabase, tlsSkipVerify, sentinelMaster) if defaultPort, ok := redisPoolFactory.CheckAndSetDefaultPort(); ok { logging.Infof("auth.redis.port isn't configured. Will be used default: %d", defaultPort) } @@ -464,10 +466,12 @@ func initializeStorage(vp *viper.Viper) (storages.ConfigurationsStorage, *meta.R port := vp.GetInt("storage.redis.port") password := vp.GetString("storage.redis.password") + database := vp.GetInt("storage.redis.database") + tlsSkipVerify := vp.GetBool("storage.redis.tls_skip_verify") sentinelMaster := vp.GetString("storage.redis.sentinel_master_name") - redisPoolFactory := meta.NewRedisPoolFactory(host, port, password, tlsSkipVerify, sentinelMaster) + redisPoolFactory := meta.NewRedisPoolFactory(host, port, password, database, tlsSkipVerify, sentinelMaster) if defaultPort, ok := redisPoolFactory.CheckAndSetDefaultPort(); ok { logging.Infof("storage.redis.port isn't configured. Will be used default: %d", defaultPort) } diff --git a/documentation/configuration/index.mdx b/documentation/configuration/index.mdx index 47919e4b7..6a0099eec 100644 --- a/documentation/configuration/index.mdx +++ b/documentation/configuration/index.mdx @@ -97,6 +97,7 @@ meta: host: redis_host port: 6379 password: secret_password + database: 0 node: pool_size: 1 # default diff --git a/documentation/deployment/scale.mdx b/documentation/deployment/scale.mdx index b190955c4..8bb98efb9 100644 --- a/documentation/deployment/scale.mdx +++ b/documentation/deployment/scale.mdx @@ -80,6 +80,7 @@ coordination: host: your_redis_host port: 6379 password: secret_password + database: 0 ``` @@ -98,6 +99,7 @@ meta.storage: host: your_redis_host port: 6379 password: secret_password + database: 0 coordination: type: redis @@ -117,6 +119,7 @@ meta: host: your_sentinel_host # eg: localhost, redis-headless.redis.svc.cluster.local port: 26379 sentinel_master_name: redis_dev_master + database: 0 ## or configuration via sentinel URL meta: @@ -131,12 +134,14 @@ storage: sentinel_master_name: 'redis_dev_master' port: 26379 tls_skip_verify: '${env.TLS_SKIP_VERIFY|false}' + database: 0 auth: redis: host: '${env.REDIS_URL}' port: 26379 sentinel_master_name: 'redis_dev_master' + database: 0 ``` Every **Jitsu Server** instance with configured coordination sends heartbeat requests every 90 seconds. diff --git a/server/drivers/redis/redis.go b/server/drivers/redis/redis.go index cdc787109..34c9b14df 100644 --- a/server/drivers/redis/redis.go +++ b/server/drivers/redis/redis.go @@ -62,7 +62,7 @@ func NewRedis(_ context.Context, sourceConfig *base.SourceConfig, collection *ba return nil, fmt.Errorf("Error casting redis port [%s] to int: %v", config.Port.String(), err) } - factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, config.TLSSkipVerify, config.SentinelMasterName) + factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, 0, config.TLSSkipVerify, config.SentinelMasterName) if defaultPort, ok := factory.CheckAndSetDefaultPort(); ok { logging.Warnf("[%s] port wasn't provided. Will be used default one: %d", sourceConfig.SourceID, defaultPort) } @@ -96,7 +96,7 @@ func TestRedis(sourceConfig *base.SourceConfig) error { return fmt.Errorf("Error casting redis port [%s] to int: %v", config.Port.String(), err) } - factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, config.TLSSkipVerify, config.SentinelMasterName) + factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, 0, config.TLSSkipVerify, config.SentinelMasterName) factory.CheckAndSetDefaultPort() pool, err := factory.Create() diff --git a/server/main.go b/server/main.go index 6b720f5dc..c488721ad 100644 --- a/server/main.go +++ b/server/main.go @@ -570,11 +570,7 @@ func initializeCoordinationService(ctx context.Context, metaStorageConfiguration } telemetry.Coordination("redis") - factory := meta.NewRedisPoolFactory(host, - coordinationRedisConfiguration.GetInt("port"), - coordinationRedisConfiguration.GetString("password"), - coordinationRedisConfiguration.GetBool("tls_skip_verify"), - coordinationRedisConfiguration.GetString("sentinel_master_name")) + factory := meta.NewRedisPoolFactory(host, coordinationRedisConfiguration.GetInt("port"), coordinationRedisConfiguration.GetString("password"), coordinationRedisConfiguration.GetInt("database"), coordinationRedisConfiguration.GetBool("tls_skip_verify"), coordinationRedisConfiguration.GetString("sentinel_master_name")) factory.CheckAndSetDefaultPort() return coordination.NewRedisService(ctx, appconfig.Instance.ServerName, factory) } @@ -601,11 +597,7 @@ func initializeEventsQueueFactory(metaStorageConfiguration *viper.Viper) (*event var eventsQueueRedisPool *meta.RedisPool var err error if redisConfigurationSource != nil && redisConfigurationSource.GetString("host") != "" { - factory := meta.NewRedisPoolFactory(redisConfigurationSource.GetString("host"), - redisConfigurationSource.GetInt("port"), - redisConfigurationSource.GetString("password"), - redisConfigurationSource.GetBool("tls_skip_verify"), - redisConfigurationSource.GetString("sentinel_master_name")) + factory := meta.NewRedisPoolFactory(redisConfigurationSource.GetString("host"), redisConfigurationSource.GetInt("port"), redisConfigurationSource.GetString("password"), redisConfigurationSource.GetInt("database"), redisConfigurationSource.GetBool("tls_skip_verify"), redisConfigurationSource.GetString("sentinel_master_name")) opts := meta.DefaultOptions opts.MaxActive = 5000 factory.WithOptions(opts) diff --git a/server/meta/redis_factory.go b/server/meta/redis_factory.go index 47c956aca..0e4cc8df3 100644 --- a/server/meta/redis_factory.go +++ b/server/meta/redis_factory.go @@ -57,6 +57,7 @@ type RedisPoolFactory struct { host string port int password string + database int sentinelMasterName string tlsSkipVerify bool @@ -64,7 +65,7 @@ type RedisPoolFactory struct { } //NewRedisPoolFactory returns filled RedisPoolFactory and removes quotes in host -func NewRedisPoolFactory(host string, port int, password string, tlsSkipVerify bool, sentinelMasterMame string) *RedisPoolFactory { +func NewRedisPoolFactory(host string, port int, password string, database int, tlsSkipVerify bool, sentinelMasterMame string) *RedisPoolFactory { host = strings.TrimPrefix(host, `"`) host = strings.TrimPrefix(host, `'`) host = strings.TrimSuffix(host, `"`) @@ -74,6 +75,7 @@ func NewRedisPoolFactory(host string, port int, password string, tlsSkipVerify b host: host, port: port, password: password, + database: database, tlsSkipVerify: tlsSkipVerify, sentinelMasterName: sentinelMasterMame, options: DefaultOptions, @@ -149,6 +151,9 @@ func (rpf *RedisPoolFactory) getSentinelAndDialFunc() (*sentinel.Sentinel, func( defaultDialWriteTimeout := redis.DialWriteTimeout(rpf.options.DefaultDialWriteTimeout) options := []redis.DialOption{defaultDialConnectTimeout, defaultDialReadTimeout, defaultDialWriteTimeout} + if rpf.database > 0 { + options = append(options, redis.DialDatabase(rpf.database)) + } // 1. redis:// redis:// if rpf.isURL() || rpf.isSecuredURL() { diff --git a/server/meta/redis_factory_test.go b/server/meta/redis_factory_test.go index 136af82ac..21725a335 100644 --- a/server/meta/redis_factory_test.go +++ b/server/meta/redis_factory_test.go @@ -81,31 +81,31 @@ func TestGetSentinelAndDialFunc(t *testing.T) { }{ { "redis URL", - NewRedisPoolFactory("redis://localhost:6379", 0, "", false, ""), + NewRedisPoolFactory("redis://localhost:6379", 0, "", 0, false, ""), false, "", }, { "redis secured URL", - NewRedisPoolFactory("rediss://localhost:6379", 0, "", false, ""), + NewRedisPoolFactory("rediss://localhost:6379", 0, "", 0, false, ""), false, "", }, { "redis sentinel URL", - NewRedisPoolFactory("sentinel://master:pas@localhost:6379", 0, "", false, ""), + NewRedisPoolFactory("sentinel://master:pas@localhost:6379", 0, "", 0, false, ""), true, "", }, { "redis plain config", - NewRedisPoolFactory("host", 0, "pass", false, ""), + NewRedisPoolFactory("host", 0, "pass", 0, false, ""), false, "", }, { "redis plain config with sentinel", - NewRedisPoolFactory("host", 0, "pass", false, "sent"), + NewRedisPoolFactory("host", 0, "pass", 0, false, "sent"), true, "", }, diff --git a/server/meta/storage.go b/server/meta/storage.go index 0f719e648..ce8be1867 100644 --- a/server/meta/storage.go +++ b/server/meta/storage.go @@ -76,9 +76,11 @@ func InitializeStorage(metaStorageConfiguration *viper.Viper) (Storage, error) { host := metaStorageConfiguration.GetString("redis.host") port := metaStorageConfiguration.GetInt("redis.port") password := metaStorageConfiguration.GetString("redis.password") + database := metaStorageConfiguration.GetInt("redis.database") + sentinelMaster := metaStorageConfiguration.GetString("redis.sentinel_master_name") tlsSkipVerify := metaStorageConfiguration.GetBool("redis.tls_skip_verify") - factory := NewRedisPoolFactory(host, port, password, tlsSkipVerify, sentinelMaster) + factory := NewRedisPoolFactory(host, port, password, database, tlsSkipVerify, sentinelMaster) factory.CheckAndSetDefaultPort() logging.Infof("🏪 Initializing meta storage redis [%s]...", factory.Details()) diff --git a/server/users/storage.go b/server/users/storage.go index 2999df47f..46012b3f9 100644 --- a/server/users/storage.go +++ b/server/users/storage.go @@ -59,11 +59,13 @@ func InitializeStorage(enabled bool, metaStorageConfiguration *viper.Viper) (Sto host := redisConfigurationSource.GetString("host") port := redisConfigurationSource.GetInt("port") password := redisConfigurationSource.GetString("password") + database := redisConfigurationSource.GetInt("database") + sentinelMaster := redisConfigurationSource.GetString("sentinel_master_name") tlsSkipVerify := redisConfigurationSource.GetBool("tls_skip_verify") anonymousEventsMinutesTTL := redisConfigurationSource.GetInt("ttl_minutes.anonymous_events") - factory := meta.NewRedisPoolFactory(host, port, password, tlsSkipVerify, sentinelMaster) + factory := meta.NewRedisPoolFactory(host, port, password, database, tlsSkipVerify, sentinelMaster) options := factory.GetOptions() options.MaxActive = 100 factory.WithOptions(options) From 058bd11e42426cf20d86f9c7d9d5fe7f6b1576db Mon Sep 17 00:00:00 2001 From: Ildar Nurislamov Date: Tue, 9 Aug 2022 11:25:01 +0300 Subject: [PATCH 2/2] release-tool.sh. Delete app-version.json before configurator build --- release-tool.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/release-tool.sh b/release-tool.sh index d20589c1f..e59a69956 100755 --- a/release-tool.sh +++ b/release-tool.sh @@ -24,6 +24,7 @@ function fail() { function build_configurator() { echo "Building Configurator UI locally.." + rm -f configurator/frontend/main/public/app-version.json rm -f configurator/backend/build/dist/configurator && rm -rf configurator/frontend/main/build && \ cd configurator/frontend/ && pnpm clean && pnpm i && CI=false ANALYTICS_KEYS='{"eventnative": "js.gpon6lmpwquappfl07tuq.ka5sxhsm08cmblny72tevi", "sentry": "https://5d29508173c04d86b31638517ebf89b3@o330694.ingest.sentry.io/6365760"}' pnpm build && \ cd ../../ || fail 'Configurator build failed'