From f345ace3baf3bd7450ffd7006eeaaf54693edecc Mon Sep 17 00:00:00 2001 From: Emad Habib Date: Wed, 10 May 2023 09:37:20 +0200 Subject: [PATCH 1/9] Install zap log and noglog Signed-off-by: Emad Habib --- go.mod | 3 +- go.sum | 2 + go/vt/logutil/vts_logger.go | 67 ++++++++++++++++++++++++++++++++ go/vt/logutil/vts_logger_test.go | 1 + 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 go/vt/logutil/vts_logger.go create mode 100644 go/vt/logutil/vts_logger_test.go diff --git a/go.mod b/go.mod index dafc96fe585..943530dfc34 100644 --- a/go.mod +++ b/go.mod @@ -102,10 +102,12 @@ require ( github.com/kr/text v0.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 + github.com/slok/noglog v0.2.0 github.com/spf13/afero v1.11.0 github.com/spf13/jwalterweatherman v1.1.0 github.com/xlab/treeprint v1.2.0 go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/sync v0.6.0 gonum.org/v1/gonum v0.14.0 @@ -183,7 +185,6 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 // indirect diff --git a/go.sum b/go.sum index 05ca79a0c00..59ad72493b1 100644 --- a/go.sum +++ b/go.sum @@ -450,6 +450,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sjmudd/stopwatch v0.1.1 h1:x45OvxFB5OtCkjvYtzRF5fWB857Jzjjk84Oyd5C5ebw= github.com/sjmudd/stopwatch v0.1.1/go.mod h1:BLw0oIQJ1YLXBO/q9ufK/SgnKBVIkC2qrm6uy78Zw6U= +github.com/slok/noglog v0.2.0 h1:1czu4l2EoJ8L92UwdSXXa1Y+c5TIjFAFm2P+mjej95E= +github.com/slok/noglog v0.2.0/go.mod h1:TfKxwpEZPT+UA83bQ6RME146k0MM4e8mwHLf6bhcGDI= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= diff --git a/go/vt/logutil/vts_logger.go b/go/vt/logutil/vts_logger.go new file mode 100644 index 00000000000..83cbe795181 --- /dev/null +++ b/go/vt/logutil/vts_logger.go @@ -0,0 +1,67 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logutil + +import ( + noglog "github.com/slok/noglog" + "go.uber.org/zap" + + "vitess.io/vitess/go/vt/log" +) + +type VTSLogger zap.SugaredLogger + +// SetVTStructureLogger in-place noglog replacement with Zap's logger. +func SetVTStructureLogger(conf *zap.Config) (vtSLogger *zap.SugaredLogger, err error) { + var l *zap.Logger + + // Use the passed configuration instead of the default configuration + if conf == nil { + defaultProdConf := zap.NewProductionConfig() + conf = &defaultProdConf + } + + // Build configuration and generate a sugared logger + l, err = conf.Build() + vtSLogger = l.Sugar() + + noglog.SetLogger(&noglog.LoggerFunc{ + DebugfFunc: func(f string, a ...interface{}) { vtSLogger.Debugf(f, a...) }, + InfofFunc: func(f string, a ...interface{}) { vtSLogger.Infof(f, a...) }, + WarnfFunc: func(f string, a ...interface{}) { vtSLogger.Warnf(f, a...) }, + ErrorfFunc: func(f string, a ...interface{}) { vtSLogger.Errorf(f, a...) }, + }) + + log.Flush = noglog.Flush + log.Info = noglog.Info + log.Infof = noglog.Infof + log.InfoDepth = noglog.InfoDepth + log.Warning = noglog.Warning + log.Warningf = noglog.Warningf + log.WarningDepth = noglog.WarningDepth + log.Error = noglog.Error + log.Errorf = noglog.Errorf + log.ErrorDepth = noglog.ErrorDepth + log.Exit = noglog.Exit + log.Exitf = noglog.Exitf + log.ExitDepth = noglog.ExitDepth + log.Fatal = noglog.Fatal + log.Fatalf = noglog.Fatalf + log.FatalDepth = noglog.FatalDepth + + return +} diff --git a/go/vt/logutil/vts_logger_test.go b/go/vt/logutil/vts_logger_test.go new file mode 100644 index 00000000000..c2baa62e441 --- /dev/null +++ b/go/vt/logutil/vts_logger_test.go @@ -0,0 +1 @@ +package logutil From 03a7cfeebeeac45580eae68e1662e5bba91673e6 Mon Sep 17 00:00:00 2001 From: Emad Habib Date: Wed, 10 May 2023 10:54:29 +0200 Subject: [PATCH 2/9] Implement the Vitess Structure Logger VTSLoger Signed-off-by: Emad Habib --- go/flags/endtoend/mysqlctl.txt | 1 + go/flags/endtoend/mysqlctld.txt | 1 + go/flags/endtoend/vtcombo.txt | 1 + go/flags/endtoend/vtctld.txt | 2 +- go/flags/endtoend/vtgate.txt | 1 + go/flags/endtoend/vtorc.txt | 92 ++++++++++++++++++++ go/flags/endtoend/vttablet.txt | 1 + go/flags/endtoend/vttestserver.txt | 2 +- go/vt/logutil/vts_logger_test.go | 129 +++++++++++++++++++++++++++++ go/vt/servenv/servenv.go | 14 +++- 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index d729a44826d..6609396705e 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -88,6 +88,7 @@ Flags: --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice --socket_file string Local unix socket file to listen on --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_uid uint32 Tablet UID. (default 41983) diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index a5bec13f09a..e46741f643b 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -114,6 +114,7 @@ Flags: --shutdown-wait-time duration How long to wait for mysqld shutdown (default 5m0s) --socket_file string Local unix socket file to listen on --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_uid uint32 Tablet UID (default 41983) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 99142bd0b74..f7e2439cad5 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -334,6 +334,7 @@ Flags: --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --stream_buffer_size int the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768) --stream_health_buffer_size uint max streaming health entries to buffer per streaming health client (default 20) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --table_gc_lifecycle string States for a DROP TABLE garbage collection cycle. Default is 'hold,purge,evac,drop', use any subset ('drop' implicitly always included) (default "hold,purge,evac,drop") --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 4cb5c1e5082..ca1c95109cf 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -130,7 +130,7 @@ Flags: --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_grpc_ca string the server ca to use to validate servers when connecting diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index f42f6a2f331..65e12b0b942 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -192,6 +192,7 @@ Flags: --statsd_sample_rate float Sample rate for statsd metrics (default 1) --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --stream_buffer_size int the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_filters strings Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch. --tablet_grpc_ca string the server ca to use to validate servers when connecting diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index f65eb12d142..007aff9a46d 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -1,3 +1,4 @@ +<<<<<<< HEAD VTOrc is the automated fault detection and repair tool in Vitess. Usage: @@ -78,6 +79,7 @@ Flags: --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting --tablet_manager_grpc_cert string the cert to use to connect @@ -113,3 +115,93 @@ Flags: -v, --version print binary version --vmodule vModuleFlag comma-separated list of pattern=N settings for file-filtered logging --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) +======= +Usage of vtorc: + --alsologtostderr log to standard error as well as files + --audit-file-location string File location where the audit logs are to be stored + --audit-purge-duration duration Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s) + --audit-to-backend Whether to store the audit log in the VTOrc database + --audit-to-syslog Whether to store the audit log in the syslog + --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" + --config string config file name + --consul_auth_static_file string JSON File to read the topos/tokens from. + --emit_stats If set, emit stats to push-based monitoring and stats backends + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --instance-poll-time duration Timer duration on which VTOrc refreshes MySQL information (default 5s) + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) + --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --max-stack-size int configure the maximum stack size in bytes (default 67108864) + --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) + --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) + --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. + --port int port for the server + --pprof strings enable profiling + --prevent-cross-cell-failover Prevent VTOrc from promoting a primary in a different cell than the current primary in case of a failover + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --reasonable-replication-lag duration Maximum replication lag on replicas which is deemed to be acceptable (default 10s) + --recovery-period-block-duration duration Duration for which a new recovery is blocked on an instance after running a recovery (default 30s) + --recovery-poll-duration duration Timer duration on which VTOrc polls its database to run a recovery (default 1s) + --remote_operation_timeout duration time to wait for a remote operation (default 15s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --shutdown_wait_time duration Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM (default 30s) + --snapshot-topology-interval duration Timer duration on which VTOrc takes a snapshot of the current MySQL information it has in the database. Should be in multiple of hours + --sqlite-data-file string SQLite Datafile to use as VTOrc's database (default "file::memory:?mode=memory&cache=shared") + --stats_backend string The name of the registered push-based monitoring/stats backend to use + --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars + --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 + --stats_drop_variables string Variables to be dropped from the list of exported variables. + --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class + --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting + --tablet_manager_grpc_cert string the cert to use to connect + --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) + --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) + --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting + --tablet_manager_grpc_key string the key to use to connect + --tablet_manager_grpc_server_name string the server name to use to validate server certificate + --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo-information-refresh-duration duration Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s) + --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) + --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") + --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) + --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) + --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server + --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS + --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS + --topo_global_root string the path of the global topology data in the global topology server + --topo_global_server_address string the address of the global topology server + --topo_implementation string the topology implementation to use + --topo_k8s_context string The kubeconfig context to use, overrides the 'current-context' from the config + --topo_k8s_kubeconfig string Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod + --topo_k8s_namespace string The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config + --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass + --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) + --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) + --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server + --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS + --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) +>>>>>>> 1540296507 (Implement the Vitess Structure Logger VTSLoger) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index dbc14039ec3..fb08bbea649 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -332,6 +332,7 @@ Flags: --statsd_sample_rate float Sample rate for statsd metrics (default 1) --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --stream_health_buffer_size uint max streaming health entries to buffer per streaming health client (default 20) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-acl-config string path to table access checker config file; send SIGHUP to reload this file --table-acl-config-reload-interval duration Ticker to reload ACLs. Duration flag, format e.g.: 30s. Default: do not reload --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index ece7ff2bcec..7760a5bd713 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -114,7 +114,7 @@ Flags: --snapshot_file string A MySQL DB snapshot file --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) - --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (Zap) or the original (glog) logger --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_hostname string The hostname to use for the tablet otherwise it will be derived from OS' hostname (default "localhost") diff --git a/go/vt/logutil/vts_logger_test.go b/go/vt/logutil/vts_logger_test.go index c2baa62e441..9240dd7323a 100644 --- a/go/vt/logutil/vts_logger_test.go +++ b/go/vt/logutil/vts_logger_test.go @@ -1 +1,130 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package logutil + +import ( + "bytes" + "encoding/json" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + vtlog "vitess.io/vitess/go/vt/log" +) + +// MemorySink implements zap.Sink by writing all messages to a buffer. +// It's used to capture the logs. +type MemorySink struct { + *bytes.Buffer +} + +// Implement Close and Sync as no-ops to satisfy the interface. The Write +// method is provided by the embedded buffer. +func (s *MemorySink) Close() error { return nil } +func (s *MemorySink) Sync() error { return nil } + +func SetupLoggerWithMemSink() (sink *MemorySink, err error) { + // Create a sink instance, and register it with zap for the "memory" protocol. + sink = &MemorySink{new(bytes.Buffer)} + err = zap.RegisterSink("memory", func(*url.URL) (zap.Sink, error) { + return sink, nil + }) + if err != nil { + return nil, err + } + + testLoggerConf := NewMemorySinkConfig() + _, err = SetVTStructureLogger(&testLoggerConf) + if err != nil { + return nil, err + } + + return +} + +func NewMemorySinkConfig() zap.Config { + return zap.Config{ + Level: zap.NewAtomicLevelAt(zap.InfoLevel), + Development: false, + Sampling: &zap.SamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + Encoding: "json", + EncoderConfig: zap.NewProductionEncoderConfig(), + OutputPaths: []string{"memory://"}, + ErrorOutputPaths: []string{"memory://"}, + } +} + +func TestVTSLogger_Replacing_glog(t *testing.T) { + type logMsg struct { + Level string `json:"level"` + Msg string `json:"msg"` + } + + type testCase struct { + name string + logLevel zapcore.Level + } + + dummyLogMessage := "testing log" + testCases := []testCase{ + {"log info", zap.InfoLevel}, + {"log warn", zap.WarnLevel}, + {"log error", zap.ErrorLevel}, + } + + sink, err := SetupLoggerWithMemSink() + assert.NoError(t, err) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var loggingFunc func(format string, args ...interface{}) + var expectedLevel string + + switch tc.logLevel { + case zapcore.InfoLevel: + loggingFunc = vtlog.Infof + expectedLevel = "info" + case zapcore.ErrorLevel: + loggingFunc = vtlog.Errorf + expectedLevel = "error" + case zapcore.WarnLevel: + loggingFunc = vtlog.Warningf + expectedLevel = "warn" + } + + loggingFunc(dummyLogMessage) + + // Unmarshal the captured log. This means we're getting a struct log. + actualLog := logMsg{} + err = json.Unmarshal(sink.Bytes(), &actualLog) + assert.NoError(t, err) + // Reset the sink so that it'll contain one log per test case. + sink.Reset() + + assert.Equal(t, expectedLevel, actualLog.Level) + assert.Equal(t, dummyLogMessage, actualLog.Msg) + + }) + } +} diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index 6a7898501f8..f5214f42815 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -82,6 +82,7 @@ var ( maxStackSize = 64 * 1024 * 1024 initStartTime time.Time // time when tablet init started: for debug purposes to time how long a tablet init takes tableRefreshInterval int + useVTSLogger bool ) // RegisterFlags installs the flags used by Init, Run, and RunDefault. @@ -98,7 +99,10 @@ func RegisterFlags() { fs.IntVar(&tableRefreshInterval, "table-refresh-interval", tableRefreshInterval, "interval in milliseconds to refresh tables in status page with refreshRequired class") // pid_file.go - fs.StringVar(&pidFile, "pid_file", pidFile, "If set, the process will write its pid to the named file, and delete it on graceful shutdown.") + fs.StringVar(&pidFile, "pid_file", pidFile, "If set, the process will write its pid to the named file, and delete it on graceful shutdown.") // Logging + + // Logging + fs.BoolVar(&useVTSLogger, "structured-logging", useVTSLogger, "whether to use structured logging (Zap) or the original (glog) logger") }) } @@ -285,6 +289,14 @@ func ParseFlags(cmd string) { os.Exit(0) } + if useVTSLogger { + // Replace glog logger with zap logger + _, err := logutil.SetVTStructureLogger(nil) + if err != nil { + log.Exitf("error while setting the Zap logger: %s", err) + } + } + args := fs.Args() if len(args) > 0 { _flag.Usage() From fbb19d31643b075170d61e7940878a1f0e06bb56 Mon Sep 17 00:00:00 2001 From: Emad Mokhtar Date: Fri, 28 Jul 2023 13:15:44 +0200 Subject: [PATCH 3/9] Move the structure logging code to logger.go file --- go/vt/logutil/logger.go | 47 +++++++++++ go/vt/logutil/logger_test.go | 106 +++++++++++++++++++++++++ go/vt/logutil/vts_logger.go | 67 ---------------- go/vt/logutil/vts_logger_test.go | 130 ------------------------------- 4 files changed, 153 insertions(+), 197 deletions(-) delete mode 100644 go/vt/logutil/vts_logger.go delete mode 100644 go/vt/logutil/vts_logger_test.go diff --git a/go/vt/logutil/logger.go b/go/vt/logutil/logger.go index 47c3f124238..a9620f536e2 100644 --- a/go/vt/logutil/logger.go +++ b/go/vt/logutil/logger.go @@ -24,7 +24,11 @@ import ( "sync" "time" + noglog "github.com/slok/noglog" + "go.uber.org/zap" + "vitess.io/vitess/go/protoutil" + "vitess.io/vitess/go/vt/log" logutilpb "vitess.io/vitess/go/vt/proto/logutil" ) @@ -383,3 +387,46 @@ func fileAndLine(depth int) (string, int64) { } return file, int64(line) } + +type VTSLogger zap.SugaredLogger + +// SetVTStructureLogger in-place noglog replacement with Zap's logger. +func SetVTStructureLogger(conf *zap.Config) (vtSLogger *zap.SugaredLogger, err error) { + var l *zap.Logger + + // Use the passed configuration instead of the default configuration + if conf == nil { + defaultProdConf := zap.NewProductionConfig() + conf = &defaultProdConf + } + + // Build configuration and generate a sugared logger + l, err = conf.Build() + vtSLogger = l.Sugar() + + noglog.SetLogger(&noglog.LoggerFunc{ + DebugfFunc: func(f string, a ...interface{}) { vtSLogger.Debugf(f, a...) }, + InfofFunc: func(f string, a ...interface{}) { vtSLogger.Infof(f, a...) }, + WarnfFunc: func(f string, a ...interface{}) { vtSLogger.Warnf(f, a...) }, + ErrorfFunc: func(f string, a ...interface{}) { vtSLogger.Errorf(f, a...) }, + }) + + log.Flush = noglog.Flush + log.Info = noglog.Info + log.Infof = noglog.Infof + log.InfoDepth = noglog.InfoDepth + log.Warning = noglog.Warning + log.Warningf = noglog.Warningf + log.WarningDepth = noglog.WarningDepth + log.Error = noglog.Error + log.Errorf = noglog.Errorf + log.ErrorDepth = noglog.ErrorDepth + log.Exit = noglog.Exit + log.Exitf = noglog.Exitf + log.ExitDepth = noglog.ExitDepth + log.Fatal = noglog.Fatal + log.Fatalf = noglog.Fatalf + log.FatalDepth = noglog.FatalDepth + + return +} diff --git a/go/vt/logutil/logger_test.go b/go/vt/logutil/logger_test.go index ce25543da5f..8d7cd61dedb 100644 --- a/go/vt/logutil/logger_test.go +++ b/go/vt/logutil/logger_test.go @@ -17,8 +17,15 @@ limitations under the License. package logutil import ( + "bytes" + "encoding/json" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "net/url" "testing" "time" + vtlog "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/protoutil" "vitess.io/vitess/go/race" @@ -152,3 +159,102 @@ func TestTeeLogger(t *testing.T) { } } } + +// MemorySink implements zap.Sink by writing all messages to a buffer. +// It's used to capture the logs. +type MemorySink struct { + *bytes.Buffer +} + +// Implement Close and Sync as no-ops to satisfy the interface. The Write +// method is provided by the embedded buffer. +func (s *MemorySink) Close() error { return nil } +func (s *MemorySink) Sync() error { return nil } + +func SetupLoggerWithMemSink() (sink *MemorySink, err error) { + // Create a sink instance, and register it with zap for the "memory" protocol. + sink = &MemorySink{new(bytes.Buffer)} + err = zap.RegisterSink("memory", func(*url.URL) (zap.Sink, error) { + return sink, nil + }) + if err != nil { + return nil, err + } + + testLoggerConf := NewMemorySinkConfig() + _, err = SetVTStructureLogger(&testLoggerConf) + if err != nil { + return nil, err + } + + return +} + +func NewMemorySinkConfig() zap.Config { + return zap.Config{ + Level: zap.NewAtomicLevelAt(zap.InfoLevel), + Development: false, + Sampling: &zap.SamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + Encoding: "json", + EncoderConfig: zap.NewProductionEncoderConfig(), + OutputPaths: []string{"memory://"}, + ErrorOutputPaths: []string{"memory://"}, + } +} + +func TestVTSLogger_Replacing_glog(t *testing.T) { + type logMsg struct { + Level string `json:"level"` + Msg string `json:"msg"` + } + + type testCase struct { + name string + logLevel zapcore.Level + } + + dummyLogMessage := "testing log" + testCases := []testCase{ + {"log info", zap.InfoLevel}, + {"log warn", zap.WarnLevel}, + {"log error", zap.ErrorLevel}, + } + + sink, err := SetupLoggerWithMemSink() + assert.NoError(t, err) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var loggingFunc func(format string, args ...interface{}) + var expectedLevel string + + switch tc.logLevel { + case zapcore.InfoLevel: + loggingFunc = vtlog.Infof + expectedLevel = "info" + case zapcore.ErrorLevel: + loggingFunc = vtlog.Errorf + expectedLevel = "error" + case zapcore.WarnLevel: + loggingFunc = vtlog.Warningf + expectedLevel = "warn" + } + + loggingFunc(dummyLogMessage) + + // Unmarshal the captured log. This means we're getting a struct log. + actualLog := logMsg{} + err = json.Unmarshal(sink.Bytes(), &actualLog) + assert.NoError(t, err) + // Reset the sink so that it'll contain one log per test case. + sink.Reset() + + assert.Equal(t, expectedLevel, actualLog.Level) + assert.Equal(t, dummyLogMessage, actualLog.Msg) + + }) + } +} diff --git a/go/vt/logutil/vts_logger.go b/go/vt/logutil/vts_logger.go deleted file mode 100644 index 83cbe795181..00000000000 --- a/go/vt/logutil/vts_logger.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2023 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package logutil - -import ( - noglog "github.com/slok/noglog" - "go.uber.org/zap" - - "vitess.io/vitess/go/vt/log" -) - -type VTSLogger zap.SugaredLogger - -// SetVTStructureLogger in-place noglog replacement with Zap's logger. -func SetVTStructureLogger(conf *zap.Config) (vtSLogger *zap.SugaredLogger, err error) { - var l *zap.Logger - - // Use the passed configuration instead of the default configuration - if conf == nil { - defaultProdConf := zap.NewProductionConfig() - conf = &defaultProdConf - } - - // Build configuration and generate a sugared logger - l, err = conf.Build() - vtSLogger = l.Sugar() - - noglog.SetLogger(&noglog.LoggerFunc{ - DebugfFunc: func(f string, a ...interface{}) { vtSLogger.Debugf(f, a...) }, - InfofFunc: func(f string, a ...interface{}) { vtSLogger.Infof(f, a...) }, - WarnfFunc: func(f string, a ...interface{}) { vtSLogger.Warnf(f, a...) }, - ErrorfFunc: func(f string, a ...interface{}) { vtSLogger.Errorf(f, a...) }, - }) - - log.Flush = noglog.Flush - log.Info = noglog.Info - log.Infof = noglog.Infof - log.InfoDepth = noglog.InfoDepth - log.Warning = noglog.Warning - log.Warningf = noglog.Warningf - log.WarningDepth = noglog.WarningDepth - log.Error = noglog.Error - log.Errorf = noglog.Errorf - log.ErrorDepth = noglog.ErrorDepth - log.Exit = noglog.Exit - log.Exitf = noglog.Exitf - log.ExitDepth = noglog.ExitDepth - log.Fatal = noglog.Fatal - log.Fatalf = noglog.Fatalf - log.FatalDepth = noglog.FatalDepth - - return -} diff --git a/go/vt/logutil/vts_logger_test.go b/go/vt/logutil/vts_logger_test.go deleted file mode 100644 index 9240dd7323a..00000000000 --- a/go/vt/logutil/vts_logger_test.go +++ /dev/null @@ -1,130 +0,0 @@ -/* -Copyright 2023 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package logutil - -import ( - "bytes" - "encoding/json" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - - vtlog "vitess.io/vitess/go/vt/log" -) - -// MemorySink implements zap.Sink by writing all messages to a buffer. -// It's used to capture the logs. -type MemorySink struct { - *bytes.Buffer -} - -// Implement Close and Sync as no-ops to satisfy the interface. The Write -// method is provided by the embedded buffer. -func (s *MemorySink) Close() error { return nil } -func (s *MemorySink) Sync() error { return nil } - -func SetupLoggerWithMemSink() (sink *MemorySink, err error) { - // Create a sink instance, and register it with zap for the "memory" protocol. - sink = &MemorySink{new(bytes.Buffer)} - err = zap.RegisterSink("memory", func(*url.URL) (zap.Sink, error) { - return sink, nil - }) - if err != nil { - return nil, err - } - - testLoggerConf := NewMemorySinkConfig() - _, err = SetVTStructureLogger(&testLoggerConf) - if err != nil { - return nil, err - } - - return -} - -func NewMemorySinkConfig() zap.Config { - return zap.Config{ - Level: zap.NewAtomicLevelAt(zap.InfoLevel), - Development: false, - Sampling: &zap.SamplingConfig{ - Initial: 100, - Thereafter: 100, - }, - Encoding: "json", - EncoderConfig: zap.NewProductionEncoderConfig(), - OutputPaths: []string{"memory://"}, - ErrorOutputPaths: []string{"memory://"}, - } -} - -func TestVTSLogger_Replacing_glog(t *testing.T) { - type logMsg struct { - Level string `json:"level"` - Msg string `json:"msg"` - } - - type testCase struct { - name string - logLevel zapcore.Level - } - - dummyLogMessage := "testing log" - testCases := []testCase{ - {"log info", zap.InfoLevel}, - {"log warn", zap.WarnLevel}, - {"log error", zap.ErrorLevel}, - } - - sink, err := SetupLoggerWithMemSink() - assert.NoError(t, err) - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var loggingFunc func(format string, args ...interface{}) - var expectedLevel string - - switch tc.logLevel { - case zapcore.InfoLevel: - loggingFunc = vtlog.Infof - expectedLevel = "info" - case zapcore.ErrorLevel: - loggingFunc = vtlog.Errorf - expectedLevel = "error" - case zapcore.WarnLevel: - loggingFunc = vtlog.Warningf - expectedLevel = "warn" - } - - loggingFunc(dummyLogMessage) - - // Unmarshal the captured log. This means we're getting a struct log. - actualLog := logMsg{} - err = json.Unmarshal(sink.Bytes(), &actualLog) - assert.NoError(t, err) - // Reset the sink so that it'll contain one log per test case. - sink.Reset() - - assert.Equal(t, expectedLevel, actualLog.Level) - assert.Equal(t, dummyLogMessage, actualLog.Msg) - - }) - } -} From 15b723ae5723bde7d2e6a5ae365f120f19c9d783 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 10 Jul 2024 01:50:43 +0200 Subject: [PATCH 4/9] make PR suggestions from vitessio/vitess#13061 Signed-off-by: Tim Vaillancourt --- go/flags/endtoend/mysqlctl.txt | 2 +- go/flags/endtoend/mysqlctld.txt | 2 +- go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vtctld.txt | 2 +- go/flags/endtoend/vtgate.txt | 2 +- go/flags/endtoend/vtorc.txt | 93 +----------------------------- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- go/vt/logutil/logger.go | 6 +- go/vt/logutil/logger_test.go | 10 ++-- go/vt/servenv/servenv.go | 8 +-- 11 files changed, 21 insertions(+), 110 deletions(-) diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index 6609396705e..cb757164d35 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -88,7 +88,7 @@ Flags: --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice --socket_file string Local unix socket file to listen on --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_uid uint32 Tablet UID. (default 41983) diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index e46741f643b..a0e09e9898d 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -114,7 +114,7 @@ Flags: --shutdown-wait-time duration How long to wait for mysqld shutdown (default 5m0s) --socket_file string Local unix socket file to listen on --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_uid uint32 Tablet UID (default 41983) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index f7e2439cad5..93db5d6e8f5 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -334,7 +334,7 @@ Flags: --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --stream_buffer_size int the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768) --stream_health_buffer_size uint max streaming health entries to buffer per streaming health client (default 20) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --table_gc_lifecycle string States for a DROP TABLE garbage collection cycle. Default is 'hold,purge,evac,drop', use any subset ('drop' implicitly always included) (default "hold,purge,evac,drop") --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index ca1c95109cf..88ab9c51c20 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -130,7 +130,7 @@ Flags: --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_grpc_ca string the server ca to use to validate servers when connecting diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 65e12b0b942..f2519ddaedb 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -192,7 +192,7 @@ Flags: --statsd_sample_rate float Sample rate for statsd metrics (default 1) --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --stream_buffer_size int the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_filters strings Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch. --tablet_grpc_ca string the server ca to use to validate servers when connecting diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index 007aff9a46d..aa0b03c63ff 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -1,4 +1,3 @@ -<<<<<<< HEAD VTOrc is the automated fault detection and repair tool in Vitess. Usage: @@ -79,7 +78,7 @@ Flags: --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting --tablet_manager_grpc_cert string the cert to use to connect @@ -115,93 +114,3 @@ Flags: -v, --version print binary version --vmodule vModuleFlag comma-separated list of pattern=N settings for file-filtered logging --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) -======= -Usage of vtorc: - --alsologtostderr log to standard error as well as files - --audit-file-location string File location where the audit logs are to be stored - --audit-purge-duration duration Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s) - --audit-to-backend Whether to store the audit log in the VTOrc database - --audit-to-syslog Whether to store the audit log in the syslog - --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified - --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" - --config string config file name - --consul_auth_static_file string JSON File to read the topos/tokens from. - --emit_stats If set, emit stats to push-based monitoring and stats backends - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --instance-poll-time duration Timer duration on which VTOrc refreshes MySQL information (default 5s) - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) - --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --max-stack-size int configure the maximum stack size in bytes (default 67108864) - --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) - --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) - --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. - --port int port for the server - --pprof strings enable profiling - --prevent-cross-cell-failover Prevent VTOrc from promoting a primary in a different cell than the current primary in case of a failover - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --reasonable-replication-lag duration Maximum replication lag on replicas which is deemed to be acceptable (default 10s) - --recovery-period-block-duration duration Duration for which a new recovery is blocked on an instance after running a recovery (default 30s) - --recovery-poll-duration duration Timer duration on which VTOrc polls its database to run a recovery (default 1s) - --remote_operation_timeout duration time to wait for a remote operation (default 15s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --shutdown_wait_time duration Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM (default 30s) - --snapshot-topology-interval duration Timer duration on which VTOrc takes a snapshot of the current MySQL information it has in the database. Should be in multiple of hours - --sqlite-data-file string SQLite Datafile to use as VTOrc's database (default "file::memory:?mode=memory&cache=shared") - --stats_backend string The name of the registered push-based monitoring/stats backend to use - --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars - --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 - --stats_drop_variables string Variables to be dropped from the list of exported variables. - --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger - --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class - --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting - --tablet_manager_grpc_cert string the cert to use to connect - --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) - --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) - --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting - --tablet_manager_grpc_key string the key to use to connect - --tablet_manager_grpc_server_name string the server name to use to validate server certificate - --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") - --topo-information-refresh-duration duration Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s) - --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) - --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") - --topo_consul_lock_session_ttl string TTL for consul session. - --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) - --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) - --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server - --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS - --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS - --topo_global_root string the path of the global topology data in the global topology server - --topo_global_server_address string the address of the global topology server - --topo_implementation string the topology implementation to use - --topo_k8s_context string The kubeconfig context to use, overrides the 'current-context' from the config - --topo_k8s_kubeconfig string Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod - --topo_k8s_namespace string The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config - --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass - --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) - --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) - --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server - --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS - --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) ->>>>>>> 1540296507 (Implement the Vitess Structure Logger VTSLoger) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index fb08bbea649..08d4266aeab 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -332,7 +332,7 @@ Flags: --statsd_sample_rate float Sample rate for statsd metrics (default 1) --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --stream_health_buffer_size uint max streaming health entries to buffer per streaming health client (default 20) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-acl-config string path to table access checker config file; send SIGHUP to reload this file --table-acl-config-reload-interval duration Ticker to reload ACLs. Duration flag, format e.g.: 30s. Default: do not reload --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 7760a5bd713..38a21e1d707 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -114,7 +114,7 @@ Flags: --snapshot_file string A MySQL DB snapshot file --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) - --structured-logging whether to use structured logging (Zap) or the original (glog) logger + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_hostname string The hostname to use for the tablet otherwise it will be derived from OS' hostname (default "localhost") diff --git a/go/vt/logutil/logger.go b/go/vt/logutil/logger.go index a9620f536e2..29b1ae02d74 100644 --- a/go/vt/logutil/logger.go +++ b/go/vt/logutil/logger.go @@ -388,10 +388,10 @@ func fileAndLine(depth int) (string, int64) { return file, int64(line) } -type VTSLogger zap.SugaredLogger +type StructuredLogger zap.SugaredLogger -// SetVTStructureLogger in-place noglog replacement with Zap's logger. -func SetVTStructureLogger(conf *zap.Config) (vtSLogger *zap.SugaredLogger, err error) { +// SetStructuredLogger in-place noglog replacement with Zap's logger. +func SetStructuredLogger(conf *zap.Config) (vtSLogger *zap.SugaredLogger, err error) { var l *zap.Logger // Use the passed configuration instead of the default configuration diff --git a/go/vt/logutil/logger_test.go b/go/vt/logutil/logger_test.go index 8d7cd61dedb..7450218fb6e 100644 --- a/go/vt/logutil/logger_test.go +++ b/go/vt/logutil/logger_test.go @@ -19,12 +19,14 @@ package logutil import ( "bytes" "encoding/json" - "github.com/stretchr/testify/assert" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" "net/url" "testing" "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + vtlog "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/protoutil" @@ -205,7 +207,7 @@ func NewMemorySinkConfig() zap.Config { } } -func TestVTSLogger_Replacing_glog(t *testing.T) { +func TestStructuredLogger_Replacing_glog(t *testing.T) { type logMsg struct { Level string `json:"level"` Msg string `json:"msg"` diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index f5214f42815..ebd69ea81e7 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -82,7 +82,7 @@ var ( maxStackSize = 64 * 1024 * 1024 initStartTime time.Time // time when tablet init started: for debug purposes to time how long a tablet init takes tableRefreshInterval int - useVTSLogger bool + useStructuredLogger bool ) // RegisterFlags installs the flags used by Init, Run, and RunDefault. @@ -102,7 +102,7 @@ func RegisterFlags() { fs.StringVar(&pidFile, "pid_file", pidFile, "If set, the process will write its pid to the named file, and delete it on graceful shutdown.") // Logging // Logging - fs.BoolVar(&useVTSLogger, "structured-logging", useVTSLogger, "whether to use structured logging (Zap) or the original (glog) logger") + fs.BoolVar(&useStructuredLogger, "structured-logging", useStructuredLogger, "enable structured logging") }) } @@ -289,9 +289,9 @@ func ParseFlags(cmd string) { os.Exit(0) } - if useVTSLogger { + if useStructuredLogger { // Replace glog logger with zap logger - _, err := logutil.SetVTStructureLogger(nil) + _, err := logutil.SetStructuredLogger(nil) if err != nil { log.Exitf("error while setting the Zap logger: %s", err) } From 27bbbff9dc37f9a8d8321c6dc256b0ab87ecf465 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 10 Jul 2024 01:52:26 +0200 Subject: [PATCH 5/9] Fix bad merge conflict Signed-off-by: Tim Vaillancourt --- go/flags/endtoend/vtctld.txt | 1 + go/flags/endtoend/vttestserver.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 88ab9c51c20..fcf3d4e266c 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -130,6 +130,7 @@ Flags: --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity Flag logs at or above this threshold go to stderr (default 1) --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 38a21e1d707..accb85a3988 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -114,6 +114,7 @@ Flags: --snapshot_file string A MySQL DB snapshot file --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) + --stderrthreshold severity Flag logs at or above this threshold go to stderr (default 1) --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. From ac21a9a664669e43ec45361ccdcab2f128b44690 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 10 Jul 2024 01:53:34 +0200 Subject: [PATCH 6/9] Fix bad merge conflict again Signed-off-by: Tim Vaillancourt --- go/flags/endtoend/vtctld.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index fcf3d4e266c..ad85915d34c 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -130,7 +130,7 @@ Flags: --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity Flag logs at or above this threshold go to stderr (default 1) + --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index accb85a3988..c94918db1ed 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -114,7 +114,7 @@ Flags: --snapshot_file string A MySQL DB snapshot file --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) - --stderrthreshold severity Flag logs at or above this threshold go to stderr (default 1) + --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. From 7009b2d7fb214970172e254339211be13be3ec2a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 10 Jul 2024 02:32:54 +0200 Subject: [PATCH 7/9] fix test Signed-off-by: Tim Vaillancourt --- go/vt/logutil/logger_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/logutil/logger_test.go b/go/vt/logutil/logger_test.go index 7450218fb6e..5f34d2f7df3 100644 --- a/go/vt/logutil/logger_test.go +++ b/go/vt/logutil/logger_test.go @@ -184,7 +184,7 @@ func SetupLoggerWithMemSink() (sink *MemorySink, err error) { } testLoggerConf := NewMemorySinkConfig() - _, err = SetVTStructureLogger(&testLoggerConf) + _, err = SetStructuredLogger(&testLoggerConf) if err != nil { return nil, err } From 831a858d938a2e41195a393fe03bf166cbc8a3b7 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 10 Jul 2024 02:42:13 +0200 Subject: [PATCH 8/9] add flags e2e test for vtgateclienttest Signed-off-by: Tim Vaillancourt --- go/flags/endtoend/vtgateclienttest.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/go/flags/endtoend/vtgateclienttest.txt b/go/flags/endtoend/vtgateclienttest.txt index 32e892805e4..9967be213a1 100644 --- a/go/flags/endtoend/vtgateclienttest.txt +++ b/go/flags/endtoend/vtgateclienttest.txt @@ -61,6 +61,7 @@ Flags: --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice --stderrthreshold severityFlag logs at or above this threshold go to stderr (default 1) + --structured-logging enable structured logging --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class --v Level log level for V logs -v, --version print binary version From 8d14c011acfda61d6c5bd07162f50b65d46e0197 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 10 Jul 2024 02:49:20 +0200 Subject: [PATCH 9/9] update error msg Signed-off-by: Tim Vaillancourt --- go/vt/servenv/servenv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index ebd69ea81e7..9d8894a7eb6 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -293,7 +293,7 @@ func ParseFlags(cmd string) { // Replace glog logger with zap logger _, err := logutil.SetStructuredLogger(nil) if err != nil { - log.Exitf("error while setting the Zap logger: %s", err) + log.Exitf("error while setting the structured logger: %s", err) } }