Skip to content

Commit

Permalink
Wire up open telemetry into the agent config updater
Browse files Browse the repository at this point in the history 
The agent config updater should now be able to write any controller
config changes to the agent config. This means we should be able to
read this in any worker if required.
  • Loading branch information
@SimonRichardson
SimonRichardson committed Aug 17, 2023
1 parent 306f6d1 commit 21933ec
Show file tree
Hide file tree
Showing 4 changed files with 342 additions and 53 deletions.
75 changes: 63 additions & 12 deletions worker/agentconfigupdater/manifold.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,28 +88,47 @@ func Manifold(config ManifoldConfig) dependency.Manifold {
return nil, errors.Annotate(err, "getting controller config")
}

logger := config.Logger
var (
logger = config.Logger
currentConfig = agent.CurrentConfig()
)

// If the mongo memory profile from the controller config
// is different from the one in the agent config we need to
// restart the agent to apply the memory profile to the mongo
// service.
agentsMongoMemoryProfile := agent.CurrentConfig().MongoMemoryProfile()
agentsMongoMemoryProfile := currentConfig.MongoMemoryProfile()
configMongoMemoryProfile := mongo.MemoryProfile(controllerConfig.MongoMemoryProfile())
mongoProfileChanged := agentsMongoMemoryProfile != configMongoMemoryProfile

agentsJujuDBSnapChannel := agent.CurrentConfig().JujuDBSnapChannel()
agentsJujuDBSnapChannel := currentConfig.JujuDBSnapChannel()
configJujuDBSnapChannel := controllerConfig.JujuDBSnapChannel()
jujuDBSnapChannelChanged := agentsJujuDBSnapChannel != configJujuDBSnapChannel

agentsQueryTracingEnabled := agent.CurrentConfig().QueryTracingEnabled()
agentsQueryTracingEnabled := currentConfig.QueryTracingEnabled()
configQueryTracingEnabled := controllerConfig.QueryTracingEnabled()
queryTracingEnabledChanged := agentsQueryTracingEnabled != configQueryTracingEnabled

agentsQueryTracingThreshold := agent.CurrentConfig().QueryTracingThreshold()
agentsQueryTracingThreshold := currentConfig.QueryTracingThreshold()
configQueryTracingThreshold := controllerConfig.QueryTracingThreshold()
queryTracingThresholdChanged := agentsQueryTracingThreshold != configQueryTracingThreshold

agentsOpenTelemetryEnabled := currentConfig.OpenTelemetryEnabled()
configOpenTelemetryEnabled := controllerConfig.OpenTelemetryEnabled()
openTelemetryEnabledChanged := agentsOpenTelemetryEnabled != configOpenTelemetryEnabled

agentsOpenTelemetryEndpoint := currentConfig.OpenTelemetryEndpoint()
configOpenTelemetryEndpoint := controllerConfig.OpenTelemetryEndpoint()
openTelemetryEndpointChanged := agentsOpenTelemetryEndpoint != configOpenTelemetryEndpoint

agentsOpenTelemetryInsecure := currentConfig.OpenTelemetryInsecure()
configOpenTelemetryInsecure := controllerConfig.OpenTelemetryInsecure()
openTelemetryInsecureChanged := agentsOpenTelemetryInsecure != configOpenTelemetryInsecure

agentsOpenTelemetryStackTraces := currentConfig.OpenTelemetryStackTraces()
configOpenTelemetryStackTraces := controllerConfig.OpenTelemetryStackTraces()
openTelemetryStackTracesChanged := agentsOpenTelemetryStackTraces != configOpenTelemetryStackTraces

info, err := apiState.StateServingInfo()
if err != nil {
return nil, errors.Annotate(err, "getting state serving info")
Expand Down Expand Up @@ -144,6 +163,22 @@ func Manifold(config ManifoldConfig) dependency.Manifold {
logger.Debugf("setting agent config query tracing threshold: %d => %d", agentsQueryTracingThreshold, configQueryTracingThreshold)
config.SetQueryTracingThreshold(configQueryTracingThreshold)
}
if openTelemetryEnabledChanged {
logger.Debugf("setting open telemetry enabled: %t => %t", agentsOpenTelemetryEnabled, configOpenTelemetryEnabled)
config.SetOpenTelemetryEnabled(configOpenTelemetryEnabled)
}
if openTelemetryEndpointChanged {
logger.Debugf("setting open telemetry endpoint: %q => %q", agentsOpenTelemetryEndpoint, configOpenTelemetryEndpoint)
config.SetOpenTelemetryEndpoint(configOpenTelemetryEndpoint)
}
if openTelemetryInsecureChanged {
logger.Debugf("setting open telemetry insecure: %t => %t", agentsOpenTelemetryInsecure, configOpenTelemetryInsecure)
config.SetOpenTelemetryInsecure(configOpenTelemetryInsecure)
}
if openTelemetryStackTracesChanged {
logger.Debugf("setting open telemetry stack trace: %t => %t", agentsOpenTelemetryStackTraces, configOpenTelemetryStackTraces)
config.SetOpenTelemetryStackTraces(configOpenTelemetryStackTraces)
}

return nil
})
Expand All @@ -164,6 +199,18 @@ func Manifold(config ManifoldConfig) dependency.Manifold {
} else if queryTracingThresholdChanged {
logger.Infof("restarting agent for new query tracing threshold")
return nil, jworker.ErrRestartAgent
} else if openTelemetryEnabledChanged {
logger.Infof("restarting agent for new open telemetry enabled")
return nil, jworker.ErrRestartAgent
} else if openTelemetryEndpointChanged {
logger.Infof("restarting agent for new open telemetry endpoint")
return nil, jworker.ErrRestartAgent
} else if openTelemetryInsecureChanged {
logger.Infof("restarting agent for new open telemetry insecure")
return nil, jworker.ErrRestartAgent
} else if openTelemetryStackTracesChanged {
logger.Infof("restarting agent for new open telemetry stack traces")
return nil, jworker.ErrRestartAgent
}

// Only get the hub if we are a controller and we haven't updated
Expand All @@ -175,13 +222,17 @@ func Manifold(config ManifoldConfig) dependency.Manifold {
}

return NewWorker(WorkerConfig{
Agent: agent,
Hub: hub,
MongoProfile: configMongoMemoryProfile,
JujuDBSnapChannel: configJujuDBSnapChannel,
QueryTracingEnabled: configQueryTracingEnabled,
QueryTracingThreshold: configQueryTracingThreshold,
Logger: config.Logger,
Agent: agent,
Hub: hub,
MongoProfile: configMongoMemoryProfile,
JujuDBSnapChannel: configJujuDBSnapChannel,
QueryTracingEnabled: configQueryTracingEnabled,
QueryTracingThreshold: configQueryTracingThreshold,
OpenTelemetryEnabled: configOpenTelemetryEnabled,
OpenTelemetryEndpoint: configOpenTelemetryEndpoint,
OpenTelemetryInsecure: configOpenTelemetryInsecure,
OpenTelemetryStackTraces: configOpenTelemetryStackTraces,
Logger: config.Logger,
})
},
}
Expand Down
70 changes: 62 additions & 8 deletions worker/agentconfigupdater/manifold_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,13 @@ func (s *AgentConfigUpdaterSuite) TestCentralHubMissing(c *gc.C) {
result := response.(*params.ControllerConfigResult)
*result = params.ControllerConfigResult{
Config: map[string]interface{}{
"mongo-memory-profile": "default",
"juju-db-snap-channel": controller.DefaultJujuDBSnapChannel,
"query-tracing-enabled": controller.DefaultQueryTracingEnabled,
"query-tracing-threshold": controller.DefaultQueryTracingThreshold,
"mongo-memory-profile": "default",
"juju-db-snap-channel": controller.DefaultJujuDBSnapChannel,
"query-tracing-enabled": controller.DefaultQueryTracingEnabled,
"query-tracing-threshold": controller.DefaultQueryTracingThreshold,
controller.OpenTelemetryEnabled: controller.DefaultOpenTelemetryEnabled,
controller.OpenTelemetryInsecure: controller.DefaultOpenTelemetryInsecure,
controller.OpenTelemetryStackTraces: controller.DefaultOpenTelemetryStackTraces,
},
}
default:
Expand Down Expand Up @@ -227,10 +230,13 @@ func (s *AgentConfigUpdaterSuite) startManifold(c *gc.C, a agent.Agent, mockAPIP
result := response.(*params.ControllerConfigResult)
*result = params.ControllerConfigResult{
Config: map[string]interface{}{
"mongo-memory-profile": "default",
"juju-db-snap-channel": controller.DefaultJujuDBSnapChannel,
"query-tracing-enabled": controller.DefaultQueryTracingEnabled,
"query-tracing-threshold": controller.DefaultQueryTracingThreshold,
"mongo-memory-profile": "default",
"juju-db-snap-channel": controller.DefaultJujuDBSnapChannel,
"query-tracing-enabled": controller.DefaultQueryTracingEnabled,
"query-tracing-threshold": controller.DefaultQueryTracingThreshold,
controller.OpenTelemetryEnabled: controller.DefaultOpenTelemetryEnabled,
controller.OpenTelemetryInsecure: controller.DefaultOpenTelemetryInsecure,
controller.OpenTelemetryStackTraces: controller.DefaultOpenTelemetryStackTraces,
},
}
default:
Expand Down Expand Up @@ -366,6 +372,18 @@ type mockConfig struct {

queryTracingThreshold time.Duration
queryTracingThresholdSet bool

openTelemetryEnabled bool
openTelemetryEnabledSet bool

openTelemetryEndpoint string
openTelemetryEndpointSet bool

openTelemetryInsecure bool
openTelemetryInsecureSet bool

openTelemetryStackTraces bool
openTelemetryStackTracesSet bool
}

func (mc *mockConfig) Tag() names.Tag {
Expand Down Expand Up @@ -433,6 +451,42 @@ func (mc *mockConfig) SetQueryTracingThreshold(threshold time.Duration) {
mc.queryTracingThresholdSet = true
}

func (mc *mockConfig) OpenTelemetryEnabled() bool {
return mc.openTelemetryEnabled
}

func (mc *mockConfig) SetOpenTelemetryEnabled(enabled bool) {
mc.openTelemetryEnabled = enabled
mc.openTelemetryEnabledSet = true
}

func (mc *mockConfig) OpenTelemetryEndpoint() string {
return mc.openTelemetryEndpoint
}

func (mc *mockConfig) SetOpenTelemetryEndpoint(endpoint string) {
mc.openTelemetryEndpoint = endpoint
mc.openTelemetryEndpointSet = true
}

func (mc *mockConfig) OpenTelemetryInsecure() bool {
return mc.openTelemetryInsecure
}

func (mc *mockConfig) SetOpenTelemetryInsecure(enabled bool) {
mc.openTelemetryInsecure = enabled
mc.openTelemetryInsecureSet = true
}

func (mc *mockConfig) OpenTelemetryStackTraces() bool {
return mc.openTelemetryStackTraces
}

func (mc *mockConfig) SetOpenTelemetryStackTraces(enabled bool) {
mc.openTelemetryStackTraces = enabled
mc.openTelemetryStackTracesSet = true
}

func (mc *mockConfig) LogDir() string {
return "log-dir"
}
84 changes: 66 additions & 18 deletions worker/agentconfigupdater/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,17 @@ import (
// WorkerConfig contains the information necessary to run
// the agent config updater worker.
type WorkerConfig struct {
Agent coreagent.Agent
Hub *pubsub.StructuredHub
MongoProfile mongo.MemoryProfile
JujuDBSnapChannel string
QueryTracingEnabled bool
QueryTracingThreshold time.Duration
Logger Logger
Agent coreagent.Agent
Hub *pubsub.StructuredHub
MongoProfile mongo.MemoryProfile
JujuDBSnapChannel string
QueryTracingEnabled bool
QueryTracingThreshold time.Duration
OpenTelemetryEnabled bool
OpenTelemetryEndpoint string
OpenTelemetryInsecure bool
OpenTelemetryStackTraces bool
Logger Logger
}

// Validate ensures that the required values are set in the structure.
Expand All @@ -46,11 +50,15 @@ func (c *WorkerConfig) Validate() error {
type agentConfigUpdater struct {
config WorkerConfig

tomb tomb.Tomb
mongoProfile mongo.MemoryProfile
jujuDBSnapChannel string
queryTracingEnabled bool
queryTracingThreshold time.Duration
tomb tomb.Tomb
mongoProfile mongo.MemoryProfile
jujuDBSnapChannel string
queryTracingEnabled bool
queryTracingThreshold time.Duration
openTelemetryEnabled bool
openTelemetryEndpoint string
openTelemetryInsecure bool
openTelemetryStackTraces bool
}

// NewWorker creates a new agent config updater worker.
Expand All @@ -61,11 +69,15 @@ func NewWorker(config WorkerConfig) (worker.Worker, error) {

started := make(chan struct{})
w := &agentConfigUpdater{
config: config,
mongoProfile: config.MongoProfile,
jujuDBSnapChannel: config.JujuDBSnapChannel,
queryTracingEnabled: config.QueryTracingEnabled,
queryTracingThreshold: config.QueryTracingThreshold,
config: config,
mongoProfile: config.MongoProfile,
jujuDBSnapChannel: config.JujuDBSnapChannel,
queryTracingEnabled: config.QueryTracingEnabled,
queryTracingThreshold: config.QueryTracingThreshold,
openTelemetryEnabled: config.OpenTelemetryEnabled,
openTelemetryEndpoint: config.OpenTelemetryEndpoint,
openTelemetryInsecure: config.OpenTelemetryInsecure,
openTelemetryStackTraces: config.OpenTelemetryStackTraces,
}
w.tomb.Go(func() error {
return w.loop(started)
Expand Down Expand Up @@ -111,7 +123,27 @@ func (w *agentConfigUpdater) onConfigChanged(topic string, data controllermsg.Co
queryTracingThreshold := data.Config.QueryTracingThreshold()
queryTracingThresholdChanged := queryTracingThreshold != w.queryTracingThreshold

changeDetected := mongoProfileChanged || jujuDBSnapChannelChanged || queryTracingEnabledChanged || queryTracingThresholdChanged
openTelemetryEnabled := data.Config.OpenTelemetryEnabled()
openTelemetryEnabledChanged := openTelemetryEnabled != w.openTelemetryEnabled

openTelemetryEndpoint := data.Config.OpenTelemetryEndpoint()
openTelemetryEndpointChanged := openTelemetryEndpoint != w.openTelemetryEndpoint

openTelemetryInsecure := data.Config.OpenTelemetryInsecure()
openTelemetryInsecureChanged := openTelemetryInsecure != w.openTelemetryInsecure

openTelemetryStackTraces := data.Config.OpenTelemetryStackTraces()
openTelemetryStackTracesChanged := openTelemetryStackTraces != w.openTelemetryStackTraces

changeDetected := mongoProfileChanged ||
jujuDBSnapChannelChanged ||
queryTracingEnabledChanged ||
queryTracingThresholdChanged ||
openTelemetryEnabledChanged ||
openTelemetryEndpointChanged ||
openTelemetryInsecureChanged ||
openTelemetryStackTracesChanged

if !changeDetected {
// Nothing to do, all good.
return
Expand All @@ -134,6 +166,22 @@ func (w *agentConfigUpdater) onConfigChanged(topic string, data controllermsg.Co
w.config.Logger.Debugf("setting agent config query tracing threshold: %v => %v", w.queryTracingThreshold, queryTracingThreshold)
setter.SetQueryTracingThreshold(queryTracingThreshold)
}
if openTelemetryEnabledChanged {
w.config.Logger.Debugf("setting agent config open telemetry enabled: %v => %v", w.openTelemetryEnabled, openTelemetryEnabled)
setter.SetOpenTelemetryEnabled(openTelemetryEnabled)
}
if openTelemetryEndpointChanged {
w.config.Logger.Debugf("setting agent config open telemetry endpoint: %v => %v", w.openTelemetryEndpoint, openTelemetryEndpoint)
setter.SetOpenTelemetryEndpoint(openTelemetryEndpoint)
}
if openTelemetryInsecureChanged {
w.config.Logger.Debugf("setting agent config open telemetry insecure: %v => %v", w.openTelemetryInsecure, openTelemetryInsecure)
setter.SetOpenTelemetryInsecure(openTelemetryInsecure)
}
if openTelemetryStackTracesChanged {
w.config.Logger.Debugf("setting agent config open telemetry stack traces: %v => %v", w.openTelemetryStackTraces, openTelemetryStackTraces)
setter.SetOpenTelemetryStackTraces(openTelemetryStackTraces)
}
return nil
})
if err != nil {
Expand Down
Loading

0 comments on commit 21933ec

Please sign in to comment.