Merge pull request juju#16120 from SimonRichardson/add-open-telemetry…

…-agent-config

juju#16120

This is the precursor work to enable open telemetry in jujud binaries.
We require that information upon start-up of an agent and the only
way currently to do that is via setting it in the agent config.

Most of this code copies the work I did for the query tracing in
dqlite. So it's relatively easy to wire it all up, it's just very long-winded.

## Checklist

- [x] Code style: imports ordered, good names, simple structure, etc
- [x] Comments saying why design decisions were made
- [x] Go unit tests, with comments saying what you're testing

## QA steps

```sh
$ juju bootstrap lxd test --build-agent
$ juju controller-config "open-telemetry-enabled=true"
$ juju controller-config "open-telemetry-endpoint=http://10.0.0.1:16686"
```

You should see the agent restart and those values set in the agent config.

agent/agent.go

@@ -295,6 +295,21 @@ type Config interface {
 	// lower the threshold, the more queries will be output. A value of 0
 	// means all queries will be output.
 	QueryTracingThreshold() time.Duration
+
+	// OpenTelemetryEnabled returns whether the open telemetry is enabled.
+	OpenTelemetryEnabled() bool
+
+	// OpenTelemetryEndpoint returns the endpoint to use for open telemetry
+	// collection.
+	OpenTelemetryEndpoint() string
+
+	// OpenTelemetryInsecure returns if the endpoint is insecure. This is useful
+	// for local/development testing
+	OpenTelemetryInsecure() bool
+
+	// OpenTelemetryStackTraces return if debug stack traces should be enabled
+	// for each span.
+	OpenTelemetryStackTraces() bool
 }
 
 type configSetterOnly interface {
@@ -348,6 +363,21 @@ type configSetterOnly interface {
 
 	// SetQueryTracingThreshold sets the threshold for query tracing.
 	SetQueryTracingThreshold(time.Duration)
+
+	// SetOpenTelemetryEnabled sets whether open telemetry is enabled.
+	SetOpenTelemetryEnabled(bool)
+
+	// SetOpenTelemetryEndpoint sets the endpoint to use for open telemetry
+	// collection.
+	SetOpenTelemetryEndpoint(string)
+
+	// SetOpenTelemetryInsecure sets if the endpoint is insecure. This is
+	// useful for local/development testing
+	SetOpenTelemetryInsecure(bool)
+
+	// SetOpenTelemetryStackTraces sets the debug stack traces should be
+	// enabled for each span.
+	SetOpenTelemetryStackTraces(bool)
 }
 
 // LogFileName returns the filename for the Agent's log file.
@@ -404,49 +434,57 @@ func (d *apiDetails) clone() *apiDetails {
 }
 
 type configInternal struct {
-	configFilePath         string
-	paths                  Paths
-	tag                    names.Tag
-	nonce                  string
-	controller             names.ControllerTag
-	model                  names.ModelTag
-	jobs                   []model.MachineJob
-	upgradedToVersion      version.Number
-	caCert                 string
-	apiDetails             *apiDetails
-	statePassword          string
-	oldPassword            string
-	servingInfo            *controller.StateServingInfo
-	loggingConfig          string
-	values                 map[string]string
-	mongoMemoryProfile     string
-	jujuDBSnapChannel      string
-	agentLogfileMaxSizeMB  int
-	agentLogfileMaxBackups int
-	queryTracingEnabled    bool
-	queryTracingThreshold  time.Duration
+	configFilePath           string
+	paths                    Paths
+	tag                      names.Tag
+	nonce                    string
+	controller               names.ControllerTag
+	model                    names.ModelTag
+	jobs                     []model.MachineJob
+	upgradedToVersion        version.Number
+	caCert                   string
+	apiDetails               *apiDetails
+	statePassword            string
+	oldPassword              string
+	servingInfo              *controller.StateServingInfo
+	loggingConfig            string
+	values                   map[string]string
+	mongoMemoryProfile       string
+	jujuDBSnapChannel        string
+	agentLogfileMaxSizeMB    int
+	agentLogfileMaxBackups   int
+	queryTracingEnabled      bool
+	queryTracingThreshold    time.Duration
+	openTelemetryEnabled     bool
+	openTelemetryEndpoint    string
+	openTelemetryInsecure    bool
+	openTelemetryStackTraces bool
 }
 
 // AgentConfigParams holds the parameters required to create
 // a new AgentConfig.
 type AgentConfigParams struct {
-	Paths                  Paths
-	Jobs                   []model.MachineJob
-	UpgradedToVersion      version.Number
-	Tag                    names.Tag
-	Password               string
-	Nonce                  string
-	Controller             names.ControllerTag
-	Model                  names.ModelTag
-	APIAddresses           []string
-	CACert                 string
-	Values                 map[string]string
-	MongoMemoryProfile     mongo.MemoryProfile
-	JujuDBSnapChannel      string
-	AgentLogfileMaxSizeMB  int
-	AgentLogfileMaxBackups int
-	QueryTracingEnabled    bool
-	QueryTracingThreshold  time.Duration
+	Paths                    Paths
+	Jobs                     []model.MachineJob
+	UpgradedToVersion        version.Number
+	Tag                      names.Tag
+	Password                 string
+	Nonce                    string
+	Controller               names.ControllerTag
+	Model                    names.ModelTag
+	APIAddresses             []string
+	CACert                   string
+	Values                   map[string]string
+	MongoMemoryProfile       mongo.MemoryProfile
+	JujuDBSnapChannel        string
+	AgentLogfileMaxSizeMB    int
+	AgentLogfileMaxBackups   int
+	QueryTracingEnabled      bool
+	QueryTracingThreshold    time.Duration
+	OpenTelemetryEnabled     bool
+	OpenTelemetryEndpoint    string
+	OpenTelemetryInsecure    bool
+	OpenTelemetryStackTraces bool
 }
 
 // NewAgentConfig returns a new config object suitable for use for a
@@ -496,22 +534,26 @@ func NewAgentConfig(configParams AgentConfigParams) (ConfigSetterWriter, error)
 	// When/if this connection is successful, apicaller worker will generate
 	// a new secure password and update this agent's config.
 	config := &configInternal{
-		paths:                  NewPathsWithDefaults(configParams.Paths),
-		jobs:                   configParams.Jobs,
-		upgradedToVersion:      configParams.UpgradedToVersion,
-		tag:                    configParams.Tag,
-		nonce:                  configParams.Nonce,
-		controller:             configParams.Controller,
-		model:                  configParams.Model,
-		caCert:                 configParams.CACert,
-		oldPassword:            configParams.Password,
-		values:                 configParams.Values,
-		mongoMemoryProfile:     configParams.MongoMemoryProfile.String(),
-		jujuDBSnapChannel:      configParams.JujuDBSnapChannel,
-		agentLogfileMaxSizeMB:  configParams.AgentLogfileMaxSizeMB,
-		agentLogfileMaxBackups: configParams.AgentLogfileMaxBackups,
-		queryTracingEnabled:    configParams.QueryTracingEnabled,
-		queryTracingThreshold:  configParams.QueryTracingThreshold,
+		paths:                    NewPathsWithDefaults(configParams.Paths),
+		jobs:                     configParams.Jobs,
+		upgradedToVersion:        configParams.UpgradedToVersion,
+		tag:                      configParams.Tag,
+		nonce:                    configParams.Nonce,
+		controller:               configParams.Controller,
+		model:                    configParams.Model,
+		caCert:                   configParams.CACert,
+		oldPassword:              configParams.Password,
+		values:                   configParams.Values,
+		mongoMemoryProfile:       configParams.MongoMemoryProfile.String(),
+		jujuDBSnapChannel:        configParams.JujuDBSnapChannel,
+		agentLogfileMaxSizeMB:    configParams.AgentLogfileMaxSizeMB,
+		agentLogfileMaxBackups:   configParams.AgentLogfileMaxBackups,
+		queryTracingEnabled:      configParams.QueryTracingEnabled,
+		queryTracingThreshold:    configParams.QueryTracingThreshold,
+		openTelemetryEnabled:     configParams.OpenTelemetryEnabled,
+		openTelemetryEndpoint:    configParams.OpenTelemetryEndpoint,
+		openTelemetryInsecure:    configParams.OpenTelemetryInsecure,
+		openTelemetryStackTraces: configParams.OpenTelemetryStackTraces,
 	}
 	if len(configParams.APIAddresses) > 0 {
 		config.apiDetails = &apiDetails{
@@ -854,6 +896,46 @@ func (c *configInternal) SetQueryTracingThreshold(v time.Duration) {
 	c.queryTracingThreshold = v
 }
 
+// OpenTelemetryEnabled implements Config.
+func (c *configInternal) OpenTelemetryEnabled() bool {
+	return c.openTelemetryEnabled
+}
+
+// SetOpenTelemetryEnabled implements configSetterOnly.
+func (c *configInternal) SetOpenTelemetryEnabled(v bool) {
+	c.openTelemetryEnabled = v
+}
+
+// OpenTelemetryEndpoint implements Config.
+func (c *configInternal) OpenTelemetryEndpoint() string {
+	return c.openTelemetryEndpoint
+}
+
+// SetOpenTelemetryEndpoint implements configSetterOnly.
+func (c *configInternal) SetOpenTelemetryEndpoint(v string) {
+	c.openTelemetryEndpoint = v
+}
+
+// OpenTelemetryInsecure implements Config.
+func (c *configInternal) OpenTelemetryInsecure() bool {
+	return c.openTelemetryInsecure
+}
+
+// SetopenTelemetryInsecure implements configSetterOnly.
+func (c *configInternal) SetOpenTelemetryInsecure(v bool) {
+	c.openTelemetryInsecure = v
+}
+
+// OpenTelemetryStackTraces implements Config.
+func (c *configInternal) OpenTelemetryStackTraces() bool {
+	return c.openTelemetryStackTraces
+}
+
+// SetopenTelemetryStackTraces implements configSetterOnly.
+func (c *configInternal) SetOpenTelemetryStackTraces(v bool) {
+	c.openTelemetryStackTraces = v
+}
+
 var validAddr = regexp.MustCompile("^.+:[0-9]+$")
 
 func checkAddrs(addrs []string, what string) error {

agent/agent_test.go

@@ -725,3 +725,51 @@ func (*suite) TestSetQueryTracingThreshold(c *gc.C) {
 	queryTracingThreshold = conf.QueryTracingThreshold()
 	c.Assert(queryTracingThreshold, gc.Equals, time.Second*10, gc.Commentf("query tracing threshold setting not updated"))
 }
+
+func (*suite) TestSetOpenTelemetryEnabled(c *gc.C) {
+	conf, err := agent.NewAgentConfig(attributeParams)
+	c.Assert(err, jc.ErrorIsNil)
+
+	queryTracingEnabled := conf.OpenTelemetryEnabled()
+	c.Assert(queryTracingEnabled, gc.Equals, attributeParams.OpenTelemetryEnabled)
+
+	conf.SetOpenTelemetryEnabled(true)
+	queryTracingEnabled = conf.OpenTelemetryEnabled()
+	c.Assert(queryTracingEnabled, gc.Equals, true, gc.Commentf("open telemetry enabled setting not updated"))
+}
+
+func (*suite) TestSetOpenTelemetryEndpoint(c *gc.C) {
+	conf, err := agent.NewAgentConfig(attributeParams)
+	c.Assert(err, jc.ErrorIsNil)
+
+	queryTracingEndpoint := conf.OpenTelemetryEndpoint()
+	c.Assert(queryTracingEndpoint, gc.Equals, attributeParams.OpenTelemetryEndpoint)
+
+	conf.SetOpenTelemetryEndpoint("http://foo.bar")
+	queryTracingEndpoint = conf.OpenTelemetryEndpoint()
+	c.Assert(queryTracingEndpoint, gc.Equals, "http://foo.bar", gc.Commentf("open telemetry endpoint setting not updated"))
+}
+
+func (*suite) TestSetOpenTelemetryInsecure(c *gc.C) {
+	conf, err := agent.NewAgentConfig(attributeParams)
+	c.Assert(err, jc.ErrorIsNil)
+
+	queryTracingInsecure := conf.OpenTelemetryInsecure()
+	c.Assert(queryTracingInsecure, gc.Equals, attributeParams.OpenTelemetryInsecure)
+
+	conf.SetOpenTelemetryInsecure(true)
+	queryTracingInsecure = conf.OpenTelemetryInsecure()
+	c.Assert(queryTracingInsecure, gc.Equals, true, gc.Commentf("open telemetry insecure setting not updated"))
+}
+
+func (*suite) TestSetOpenTelemetryStackTraces(c *gc.C) {
+	conf, err := agent.NewAgentConfig(attributeParams)
+	c.Assert(err, jc.ErrorIsNil)
+
+	queryTracingStackTraces := conf.OpenTelemetryStackTraces()
+	c.Assert(queryTracingStackTraces, gc.Equals, attributeParams.OpenTelemetryStackTraces)
+
+	conf.SetOpenTelemetryStackTraces(true)
+	queryTracingStackTraces = conf.OpenTelemetryStackTraces()
+	c.Assert(queryTracingStackTraces, gc.Equals, true, gc.Commentf("open telemetry stack traces setting not updated"))
+}

agent/format-2.0.go

@@ -66,6 +66,11 @@ type format_2_0Serialization struct {
 	JujuDBSnapChannel     string        `yaml:"juju-db-snap-channel,omitempty"`
 	QueryTracingEnabled   bool          `yaml:"querytracingenabled,omitempty"`
 	QueryTracingThreshold time.Duration `yaml:"querytracingthreshold,omitempty"`
+
+	OpenTelemetryEnabled     bool   `yaml:"opentelemetryenabled,omitempty"`
+	OpenTelemetryEndpoint    string `yaml:"opentelemetryendpoint,omitempty"`
+	OpenTelemetryInsecure    bool   `yaml:"opentelemetryinsecure,omitempty"`
+	OpenTelemetryStackTraces bool   `yaml:"opentelemetrystacktraces,omitempty"`
 }
 
 func init() {
@@ -119,11 +124,15 @@ func (formatter_2_0) unmarshal(data []byte) (*configInternal, error) {
 
 		queryTracingEnabled:   format.QueryTracingEnabled,
 		queryTracingThreshold: format.QueryTracingThreshold,
+
+		openTelemetryEnabled:     format.OpenTelemetryEnabled,
+		openTelemetryInsecure:    format.OpenTelemetryInsecure,
+		openTelemetryStackTraces: format.OpenTelemetryStackTraces,
 	}
 	if len(format.APIAddresses) > 0 {
 		config.apiDetails = &apiDetails{
-			format.APIAddresses,
-			format.APIPassword,
+			addresses: format.APIAddresses,
+			password:  format.APIPassword,
 		}
 	}
 	if len(format.ControllerKey) != 0 {
@@ -162,6 +171,9 @@ func (formatter_2_0) unmarshal(data []byte) (*configInternal, error) {
 	if format.JujuDBSnapChannel != "" {
 		config.jujuDBSnapChannel = format.JujuDBSnapChannel
 	}
+	if format.OpenTelemetryEndpoint != "" {
+		config.openTelemetryEndpoint = format.OpenTelemetryEndpoint
+	}
 	return config, nil
 }
 
@@ -189,6 +201,10 @@ func (formatter_2_0) marshal(config *configInternal) ([]byte, error) {
 
 		QueryTracingEnabled:   config.queryTracingEnabled,
 		QueryTracingThreshold: config.queryTracingThreshold,
+
+		OpenTelemetryEnabled:     config.openTelemetryEnabled,
+		OpenTelemetryInsecure:    config.openTelemetryInsecure,
+		OpenTelemetryStackTraces: config.openTelemetryStackTraces,
 	}
 	if config.servingInfo != nil {
 		format.ControllerCert = config.servingInfo.Cert
@@ -211,5 +227,8 @@ func (formatter_2_0) marshal(config *configInternal) ([]byte, error) {
 	if config.jujuDBSnapChannel != "" {
 		format.JujuDBSnapChannel = config.jujuDBSnapChannel
 	}
+	if config.openTelemetryEndpoint != "" {
+		format.OpenTelemetryEndpoint = config.openTelemetryEndpoint
+	}
 	return goyaml.Marshal(format)
 }

agent/format-2.0_whitebox_test.go

@@ -86,6 +86,28 @@ func (*format_2_0Suite) TestQueryTracing(c *gc.C) {
 	c.Check(newConfig.QueryTracingThreshold(), gc.Equals, time.Second)
 }
 
+func (*format_2_0Suite) TestOpenTelemetry(c *gc.C) {
+	config := newTestConfig(c)
+	// configFilePath is not serialized as it is the location of the file.
+	config.configFilePath = ""
+
+	config.SetOpenTelemetryEnabled(true)
+	config.SetOpenTelemetryEndpoint("http://foo.bar")
+	config.SetOpenTelemetryInsecure(true)
+	config.SetOpenTelemetryStackTraces(true)
+
+	data, err := format_2_0.marshal(config)
+	c.Assert(err, jc.ErrorIsNil)
+	newConfig, err := format_2_0.unmarshal(data)
+	c.Assert(err, jc.ErrorIsNil)
+
+	c.Check(newConfig, gc.DeepEquals, config)
+	c.Check(newConfig.OpenTelemetryEnabled(), jc.IsTrue)
+	c.Check(newConfig.OpenTelemetryEndpoint(), gc.Equals, "http://foo.bar")
+	c.Check(newConfig.OpenTelemetryInsecure(), jc.IsTrue)
+	c.Check(newConfig.OpenTelemetryStackTraces(), jc.IsTrue)
+}
+
 var agentConfig2_0Contents = `
 # format 2.0
 controller: controller-deadbeef-1bad-500d-9000-4b1d0d06f00d

cloudconfig/instancecfg/instancecfg.go

@@ -500,6 +500,10 @@ func (cfg *InstanceConfig) AgentConfig(
 		configParams.AgentLogfileMaxSizeMB = cfg.ControllerConfig.AgentLogfileMaxSizeMB()
 		configParams.QueryTracingEnabled = cfg.ControllerConfig.QueryTracingEnabled()
 		configParams.QueryTracingThreshold = cfg.ControllerConfig.QueryTracingThreshold()
+		configParams.OpenTelemetryEnabled = cfg.ControllerConfig.OpenTelemetryEnabled()
+		configParams.OpenTelemetryEndpoint = cfg.ControllerConfig.OpenTelemetryEndpoint()
+		configParams.OpenTelemetryInsecure = cfg.ControllerConfig.OpenTelemetryInsecure()
+		configParams.OpenTelemetryStackTraces = cfg.ControllerConfig.OpenTelemetryStackTraces()
 	}
 	if cfg.Bootstrap == nil {
 		return agent.NewAgentConfig(configParams)