From 2dfc4d3690810569354b9ffabf8d619afbcb8575 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 16 Apr 2024 23:36:28 +0200 Subject: [PATCH] `slack-vitess-r15.0.5`: forward-port consul topo limits PR #111 (#297) * `slack-vitess-r14.0.5`: allow conn overrides in consul topo (#111) * `slack-vitess-r14.0.5`: allow conn overrides in consul topo Signed-off-by: Tim Vaillancourt * fix e2e test Signed-off-by: Tim Vaillancourt --------- Signed-off-by: Tim Vaillancourt * Update flags tests that didn't exist in v14 Signed-off-by: Tim Vaillancourt --------- Signed-off-by: Tim Vaillancourt --- go/flags/endtoend/vtbackup.txt | 3 +++ go/flags/endtoend/vtctld.txt | 3 +++ go/flags/endtoend/vtgate.txt | 3 +++ go/flags/endtoend/vtorc.txt | 3 +++ go/flags/endtoend/vttablet.txt | 3 +++ go/flags/endtoend/vttestserver.txt | 3 +++ go/vt/topo/consultopo/server.go | 6 +++++- 7 files changed, 23 insertions(+), 1 deletion(-) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 002f4af2831..0d2c4e968ec 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -213,9 +213,12 @@ Flags: --tablet_manager_grpc_key string the key to use to connect --tablet_manager_grpc_server_name string the server name to use to validate server certificate --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo_consul_idle_conn_timeout duration Maximum amount of time to pool idle connections. (default 1m30s) --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_max_conns_per_host int Maximum number of consul connections per host. + --topo_consul_max_idle_conns int Maximum number of idle consul connections. (default 100) --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 6056d16c4ac..85dfe19b846 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -151,9 +151,12 @@ Flags: --tablet_refresh_interval duration Tablet refresh interval. (default 1m0s) --tablet_refresh_known_tablets Whether to reload the tablet's address/port map from topo in case they change. (default true) --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}") + --topo_consul_idle_conn_timeout duration Maximum amount of time to pool idle connections. (default 1m30s) --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_max_conns_per_host int Maximum number of consul connections per host. + --topo_consul_max_idle_conns int Maximum number of idle consul connections. (default 100) --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 12f83bcbcee..bdb0bedd447 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -204,9 +204,12 @@ Flags: --tablet_refresh_known_tablets Whether to reload the tablet's address/port map from topo in case they change. (default true) --tablet_types_to_wait strings Wait till connected for specified tablet types during Gateway initialization. Should be provided as a comma-separated set of tablet types. --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}") + --topo_consul_idle_conn_timeout duration Maximum amount of time to pool idle connections. (default 1m30s) --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_max_conns_per_host int Maximum number of consul connections per host. + --topo_consul_max_idle_conns int Maximum number of idle consul connections. (default 100) --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index acb69f43c8a..27c23031784 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -89,9 +89,12 @@ Flags: --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") --tolerable-replication-lag duration Amount of replication lag that is considered acceptable for a tablet to be eligible for promotion when Vitess makes the choice of a new primary in PRS --topo-information-refresh-duration duration Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s) + --topo_consul_idle_conn_timeout duration Maximum amount of time to pool idle connections. (default 1m30s) --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_max_conns_per_host int Maximum number of consul connections per host. + --topo_consul_max_idle_conns int Maximum number of idle consul connections. (default 100) --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 7eae51c5f4f..6db195ad577 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -355,9 +355,12 @@ Flags: --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") --tablet_protocol string Protocol to use to make queryservice RPCs to vttablets. (default "grpc") --throttle_tablet_types string Comma separated VTTablet types to be considered by the throttler. default: 'replica'. example: 'replica,rdonly'. 'replica' always implicitly included (default "replica") + --topo_consul_idle_conn_timeout duration Maximum amount of time to pool idle connections. (default 1m30s) --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_max_conns_per_host int Maximum number of consul connections per host. + --topo_consul_max_idle_conns int Maximum number of idle consul connections. (default 100) --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 72d9d0bbf78..865705b9e71 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -127,9 +127,12 @@ Flags: --tablet_manager_grpc_server_name string the server name to use to validate server certificate --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") --tablet_refresh_interval duration Interval at which vtgate refreshes tablet information from topology server. (default 10s) + --topo_consul_idle_conn_timeout duration Maximum amount of time to pool idle connections. (default 1m30s) --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_max_conns_per_host int Maximum number of consul connections per host. + --topo_consul_max_idle_conns int Maximum number of idle consul connections. (default 100) --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) diff --git a/go/vt/topo/consultopo/server.go b/go/vt/topo/consultopo/server.go index a7a5446c274..72b66710051 100644 --- a/go/vt/topo/consultopo/server.go +++ b/go/vt/topo/consultopo/server.go @@ -37,6 +37,7 @@ import ( var ( consulAuthClientStaticFile string + consulConfig = api.DefaultConfig() // serfHealth is the default check from consul consulLockSessionChecks = "serfHealth" consulLockSessionTTL string @@ -52,6 +53,9 @@ func registerServerFlags(fs *pflag.FlagSet) { fs.StringVar(&consulLockSessionChecks, "topo_consul_lock_session_checks", consulLockSessionChecks, "List of checks for consul session.") fs.StringVar(&consulLockSessionTTL, "topo_consul_lock_session_ttl", consulLockSessionTTL, "TTL for consul session.") fs.DurationVar(&consulLockDelay, "topo_consul_lock_delay", consulLockDelay, "LockDelay for consul session.") + fs.IntVar(&consulConfig.Transport.MaxConnsPerHost, "topo_consul_max_conns_per_host", consulConfig.Transport.MaxConnsPerHost, "Maximum number of consul connections per host.") + fs.IntVar(&consulConfig.Transport.MaxIdleConns, "topo_consul_max_idle_conns", consulConfig.Transport.MaxIdleConns, "Maximum number of idle consul connections.") + fs.DurationVar(&consulConfig.Transport.IdleConnTimeout, "topo_consul_idle_conn_timeout", consulConfig.Transport.IdleConnTimeout, "Maximum amount of time to pool idle connections.") } // ClientAuthCred credential to use for consul clusters @@ -130,7 +134,7 @@ func NewServer(cell, serverAddr, root string) (*Server, error) { if err != nil { return nil, err } - cfg := api.DefaultConfig() + cfg := consulConfig cfg.Address = serverAddr if creds != nil { if creds[cell] != nil {