From 8bdf82055d98a7da874dc63408b59d39c5a0f810 Mon Sep 17 00:00:00 2001 From: Nikodem Rafalski Date: Thu, 15 Feb 2024 13:01:18 +0100 Subject: [PATCH] feat: Azure Monitor Logs SLOs support [PC-11534] (#143) Co-authored-by: bsski --- Makefile | 2 +- README.md | 2 +- cspell.yaml | 1 + docs/index.md | 2 +- docs/resources/slo.md | 88 ++++++++++++++++++------ examples/provider/provider.tf | 2 +- nobl9/resource_slo.go | 125 +++++++++++++++++++++++----------- nobl9/resource_slo_test.go | 72 +++++++++++++++++++- 8 files changed, 229 insertions(+), 65 deletions(-) diff --git a/Makefile b/Makefile index 717682a8..90bd722a 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ NAMESPACE=nobl9 NAME=nobl9 BIN_DIR=./bin BINARY=$(BIN_DIR)/terraform-provider-$(NAME) -VERSION=0.23.0-beta +VERSION=0.24.0 BUILD_FLAGS="-X github.com/nobl9/terraform-provider-nobl9/nobl9.Version=$(VERSION)" OS_ARCH?=linux_amd64 diff --git a/README.md b/README.md index 977fa496..994128a4 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ terraform { required_providers { nobl9 = { source = "nobl9/nobl9" - version = "0.23.0-beta" + version = "0.24.0" } } } diff --git a/cspell.yaml b/cspell.yaml index 30c911c9..3e22713d 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -36,6 +36,7 @@ words: - gopath - gosec - govulncheck + - kusto - ldflags - logql - msteams diff --git a/docs/index.md b/docs/index.md index 9e25428f..2c2a3115 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,7 +37,7 @@ terraform { required_providers { nobl9 = { source = "nobl9/nobl9" - version = "0.23.0-beta" + version = "0.24.0" } } } diff --git a/docs/resources/slo.md b/docs/resources/slo.md index 8f86481d..20d05a6d 100644 --- a/docs/resources/slo.md +++ b/docs/resources/slo.md @@ -261,15 +261,17 @@ Required: Required: -- `aggregation` (String) Aggregation type - `data_type` (String) Specifies source: 'metrics' or 'logs' -- `metric_name` (String) Name of the metric -- `resource_id` (String) Name of the added application Optional: -- `dimensions` (Block Set) Dimensions of the metric (see [below for nested schema](#nestedblock--objective--count_metrics--total--azure_monitor--dimensions)) -- `metric_namespace` (String) Namespace of the metric +- `aggregation` (String) Aggregation type [Required for metrics] +- `dimensions` (Block Set) Dimensions of the metric [Optional for metrics] (see [below for nested schema](#nestedblock--objective--count_metrics--total--azure_monitor--dimensions)) +- `kql_query` (String) Logs query in Kusto Query Language [Required for logs] +- `metric_name` (String) Name of the metric [Required for metrics] +- `metric_namespace` (String) Namespace of the metric [Optional for metrics] +- `resource_id` (String) Identifier of the Azure Cloud resource [Required for metrics] +- `workspace` (Block Set) Log analytics workspace [Required for logs] (see [below for nested schema](#nestedblock--objective--count_metrics--total--azure_monitor--workspace)) ### Nested Schema for `objective.count_metrics.total.azure_monitor.dimensions` @@ -280,6 +282,16 @@ Required: - `value` (String) Value + +### Nested Schema for `objective.count_metrics.total.azure_monitor.workspace` + +Required: + +- `resource_group` (String) Resource group of the workspace +- `subscription_id` (String) Subscription ID of the workspace +- `workspace_id` (String) ID of the workspace + + ### Nested Schema for `objective.count_metrics.total.bigquery` @@ -598,15 +610,17 @@ Required: Required: -- `aggregation` (String) Aggregation type - `data_type` (String) Specifies source: 'metrics' or 'logs' -- `metric_name` (String) Name of the metric -- `resource_id` (String) Name of the added application Optional: -- `dimensions` (Block Set) Dimensions of the metric (see [below for nested schema](#nestedblock--objective--count_metrics--bad--azure_monitor--dimensions)) -- `metric_namespace` (String) Namespace of the metric +- `aggregation` (String) Aggregation type [Required for metrics] +- `dimensions` (Block Set) Dimensions of the metric [Optional for metrics] (see [below for nested schema](#nestedblock--objective--count_metrics--bad--azure_monitor--dimensions)) +- `kql_query` (String) Logs query in Kusto Query Language [Required for logs] +- `metric_name` (String) Name of the metric [Required for metrics] +- `metric_namespace` (String) Namespace of the metric [Optional for metrics] +- `resource_id` (String) Identifier of the Azure Cloud resource [Required for metrics] +- `workspace` (Block Set) Log analytics workspace [Required for logs] (see [below for nested schema](#nestedblock--objective--count_metrics--bad--azure_monitor--workspace)) ### Nested Schema for `objective.count_metrics.bad.azure_monitor.dimensions` @@ -617,6 +631,16 @@ Required: - `value` (String) Value + +### Nested Schema for `objective.count_metrics.bad.azure_monitor.workspace` + +Required: + +- `resource_group` (String) Resource group of the workspace +- `subscription_id` (String) Subscription ID of the workspace +- `workspace_id` (String) ID of the workspace + + ### Nested Schema for `objective.count_metrics.bad.bigquery` @@ -935,15 +959,17 @@ Required: Required: -- `aggregation` (String) Aggregation type - `data_type` (String) Specifies source: 'metrics' or 'logs' -- `metric_name` (String) Name of the metric -- `resource_id` (String) Name of the added application Optional: -- `dimensions` (Block Set) Dimensions of the metric (see [below for nested schema](#nestedblock--objective--count_metrics--good--azure_monitor--dimensions)) -- `metric_namespace` (String) Namespace of the metric +- `aggregation` (String) Aggregation type [Required for metrics] +- `dimensions` (Block Set) Dimensions of the metric [Optional for metrics] (see [below for nested schema](#nestedblock--objective--count_metrics--good--azure_monitor--dimensions)) +- `kql_query` (String) Logs query in Kusto Query Language [Required for logs] +- `metric_name` (String) Name of the metric [Required for metrics] +- `metric_namespace` (String) Namespace of the metric [Optional for metrics] +- `resource_id` (String) Identifier of the Azure Cloud resource [Required for metrics] +- `workspace` (Block Set) Log analytics workspace [Required for logs] (see [below for nested schema](#nestedblock--objective--count_metrics--good--azure_monitor--workspace)) ### Nested Schema for `objective.count_metrics.good.azure_monitor.dimensions` @@ -954,6 +980,16 @@ Required: - `value` (String) Value + +### Nested Schema for `objective.count_metrics.good.azure_monitor.workspace` + +Required: + +- `resource_group` (String) Resource group of the workspace +- `subscription_id` (String) Subscription ID of the workspace +- `workspace_id` (String) ID of the workspace + + ### Nested Schema for `objective.count_metrics.good.bigquery` @@ -1280,15 +1316,17 @@ Required: Required: -- `aggregation` (String) Aggregation type - `data_type` (String) Specifies source: 'metrics' or 'logs' -- `metric_name` (String) Name of the metric -- `resource_id` (String) Name of the added application Optional: -- `dimensions` (Block Set) Dimensions of the metric (see [below for nested schema](#nestedblock--objective--raw_metric--query--azure_monitor--dimensions)) -- `metric_namespace` (String) Namespace of the metric +- `aggregation` (String) Aggregation type [Required for metrics] +- `dimensions` (Block Set) Dimensions of the metric [Optional for metrics] (see [below for nested schema](#nestedblock--objective--raw_metric--query--azure_monitor--dimensions)) +- `kql_query` (String) Logs query in Kusto Query Language [Required for logs] +- `metric_name` (String) Name of the metric [Required for metrics] +- `metric_namespace` (String) Namespace of the metric [Optional for metrics] +- `resource_id` (String) Identifier of the Azure Cloud resource [Required for metrics] +- `workspace` (Block Set) Log analytics workspace [Required for logs] (see [below for nested schema](#nestedblock--objective--raw_metric--query--azure_monitor--workspace)) ### Nested Schema for `objective.raw_metric.query.azure_monitor.dimensions` @@ -1299,6 +1337,16 @@ Required: - `value` (String) Value + +### Nested Schema for `objective.raw_metric.query.azure_monitor.workspace` + +Required: + +- `resource_group` (String) Resource group of the workspace +- `subscription_id` (String) Subscription ID of the workspace +- `workspace_id` (String) ID of the workspace + + ### Nested Schema for `objective.raw_metric.query.bigquery` diff --git a/examples/provider/provider.tf b/examples/provider/provider.tf index 13d14e40..13d692d2 100644 --- a/examples/provider/provider.tf +++ b/examples/provider/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { nobl9 = { source = "nobl9/nobl9" - version = "0.23.0-beta" + version = "0.24.0" } } } diff --git a/nobl9/resource_slo.go b/nobl9/resource_slo.go index 6a891b60..f588c7b3 100644 --- a/nobl9/resource_slo.go +++ b/nobl9/resource_slo.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" v1alphaSLO "github.com/nobl9/nobl9-go/manifest/v1alpha/slo" v1Objects "github.com/nobl9/nobl9-go/sdk/endpoints/objects/v1" @@ -1078,34 +1079,40 @@ func schemaMetricAzureMonitor() map[string]*schema.Schema { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "data_type": { - Type: schema.TypeString, - Required: true, + Type: schema.TypeString, + Required: true, + ValidateDiagFunc: validation.ToDiagFunc( + validation.StringInSlice([]string{ + v1alphaSLO.AzureMonitorDataTypeMetrics, + v1alphaSLO.AzureMonitorDataTypeLogs, + }, false), + ), Description: "Specifies source: 'metrics' or 'logs'", }, "resource_id": { Type: schema.TypeString, - Required: true, - Description: "Name of the added application", + Optional: true, + Description: "Identifier of the Azure Cloud resource [Required for metrics]", }, "metric_namespace": { Type: schema.TypeString, Optional: true, - Description: "Namespace of the metric", + Description: "Namespace of the metric [Optional for metrics]", }, "metric_name": { Type: schema.TypeString, - Required: true, - Description: "Name of the metric", + Optional: true, + Description: "Name of the metric [Required for metrics]", }, "aggregation": { Type: schema.TypeString, - Required: true, - Description: "Aggregation type", + Optional: true, + Description: "Aggregation type [Required for metrics]", }, "dimensions": { Type: schema.TypeSet, Optional: true, - Description: "Dimensions of the metric", + Description: "Dimensions of the metric [Optional for metrics]", MinItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -1122,6 +1129,35 @@ func schemaMetricAzureMonitor() map[string]*schema.Schema { }, }, }, + "workspace": { + Type: schema.TypeSet, + Optional: true, + Description: "Log analytics workspace [Required for logs]", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "subscription_id": { + Type: schema.TypeString, + Required: true, + Description: "Subscription ID of the workspace", + }, + "resource_group": { + Type: schema.TypeString, + Required: true, + Description: "Resource group of the workspace", + }, + "workspace_id": { + Type: schema.TypeString, + Required: true, + Description: "ID of the workspace", + }, + }, + }, + }, + "kql_query": { + Type: schema.TypeString, + Optional: true, + Description: "Logs query in Kusto Query Language [Required for logs]", + }, }, }, }, @@ -1134,38 +1170,41 @@ func marshalAzureMonitorMetric(s *schema.Set) *v1alphaSLO.AzureMonitorMetric { } metric := s.List()[0].(map[string]interface{}) - dataType := metric["data_type"].(string) - resourceID := metric["resource_id"].(string) - metricNamespace := metric["metric_namespace"].(string) - metricName := metric["metric_name"].(string) - aggregation := metric["aggregation"].(string) - - dimensions := metric["dimensions"].(*schema.Set) - var metricDimensions []v1alphaSLO.AzureMonitorMetricDimension - - if dimensions.Len() > 0 { - metricDimensions = make([]v1alphaSLO.AzureMonitorMetricDimension, dimensions.Len()) - } - - for idx, dimension := range dimensions.List() { - n9Dimension := dimension.(map[string]interface{}) - name := n9Dimension["name"].(string) - value := n9Dimension["value"].(string) - metricDimensions[idx] = v1alphaSLO.AzureMonitorMetricDimension{ - Name: &name, - Value: &value, + result := v1alphaSLO.AzureMonitorMetric{DataType: dataType} + if dataType == v1alphaSLO.AzureMonitorDataTypeMetrics { + result.ResourceID = metric["resource_id"].(string) + result.MetricNamespace = metric["metric_namespace"].(string) + result.MetricName = metric["metric_name"].(string) + result.Aggregation = metric["aggregation"].(string) + dimensions := metric["dimensions"].(*schema.Set) + var metricDimensions []v1alphaSLO.AzureMonitorMetricDimension + if dimensions.Len() > 0 { + metricDimensions = make([]v1alphaSLO.AzureMonitorMetricDimension, dimensions.Len()) + } + for idx, dimension := range dimensions.List() { + n9Dimension := dimension.(map[string]interface{}) + name := n9Dimension["name"].(string) + value := n9Dimension["value"].(string) + metricDimensions[idx] = v1alphaSLO.AzureMonitorMetricDimension{ + Name: &name, + Value: &value, + } + } + result.Dimensions = metricDimensions + } else if dataType == v1alphaSLO.AzureMonitorDataTypeLogs { + result.KQLQuery = metric["kql_query"].(string) + wsList := metric["workspace"].(*schema.Set).List() + if len(wsList) > 0 { + workspace := wsList[0].(map[string]interface{}) + result.Workspace = &v1alphaSLO.AzureMonitorMetricLogAnalyticsWorkspace{ + SubscriptionID: workspace["subscription_id"].(string), + ResourceGroup: workspace["resource_group"].(string), + WorkspaceID: workspace["workspace_id"].(string), + } } } - - return &v1alphaSLO.AzureMonitorMetric{ - DataType: dataType, - ResourceID: resourceID, - MetricNamespace: metricNamespace, - MetricName: metricName, - Aggregation: aggregation, - Dimensions: metricDimensions, - } + return &result } func unmarshalAzureMonitorMetric(metric interface{}) map[string]interface{} { @@ -1183,6 +1222,14 @@ func unmarshalAzureMonitorMetric(metric interface{}) map[string]interface{} { var dimensions any _ = json.Unmarshal(dim, &dimensions) res["dimensions"] = dimensions + res["kql_query"] = amMetric.KQLQuery + if amMetric.Workspace != nil { + res["workspace"] = schema.NewSet(oneElementSet, []interface{}{map[string]interface{}{ + "subscription_id": amMetric.Workspace.SubscriptionID, + "resource_group": amMetric.Workspace.ResourceGroup, + "workspace_id": amMetric.Workspace.WorkspaceID, + }}) + } return res } diff --git a/nobl9/resource_slo_test.go b/nobl9/resource_slo_test.go index 03420662..4bcabd43 100644 --- a/nobl9/resource_slo_test.go +++ b/nobl9/resource_slo_test.go @@ -19,7 +19,8 @@ func TestAcc_Nobl9SLO(t *testing.T) { }{ {"test-amazonprometheus", testAmazonPrometheusSLO}, {"test-appdynamics", testAppdynamicsSLO}, - {"test-azure-monitor", testAzureMonitorSLO}, + {"test-azure-monitor-metrics", testAzureMonitorMetricsSLO}, + {"test-azure-monitor-logs", testAzureMonitorLogsSLO}, {"test-bigquery", testBigQuerySLO}, {"test-cloudwatch-with-json", testCloudWatchWithJSON}, {"test-cloudwatch-with-sql", testCloudWatchWithSQL}, @@ -233,7 +234,7 @@ resource "nobl9_slo" ":name" { } // nolint: lll -func testAzureMonitorSLO(name string) string { +func testAzureMonitorMetricsSLO(name string) string { var serviceName = name + "-tf-service" var agentName = name + "-tf-agent" config := @@ -301,6 +302,73 @@ resource "nobl9_slo" ":name" { return config } +// nolint: lll +func testAzureMonitorLogsSLO(name string) string { + var serviceName = name + "-tf-service" + var agentName = name + "-tf-agent" + config := + testService(serviceName) + + testAzureMonitorAgent(agentName) + ` +resource "nobl9_slo" ":name" { + name = ":name" + display_name = ":name" + project = ":project" + service = nobl9_service.:serviceName.name + + label { + key = "team" + values = ["green","sapphire"] + } + + label { + key = "env" + values = ["dev", "staging", "prod"] + } + + budgeting_method = "Occurrences" + + objective { + display_name = "obj1" + name = "tf-objective-1" + target = 0.7 + value = 1 + op = "lt" + raw_metric { + query { + azure_monitor { + data_type = "logs" + workspace { + subscription_id = "9c26f90e-24bb-4d20-a648-c6e3e1cde26a" + resource_group = "azure-monitor-test-sources" + workspace_id = "e5da9ba8-cb8f-437e-aec0-61d21aab2bcd" + } + kql_query = "AppRequests | where AppRoleName == \"n9-web-app\" | summarize n9_value = avg(DurationMs) by bin(TimeGenerated, 15s) | project n9_time = TimeGenerated, n9_value" + } + } + } + } + + time_window { + count = 10 + is_rolling = true + unit = "Minute" + } + + indicator { + name = nobl9_agent.:agentName.name + project = ":project" + kind = "Agent" + } +} +` + config = strings.ReplaceAll(config, ":name", name) + config = strings.ReplaceAll(config, ":serviceName", serviceName) + config = strings.ReplaceAll(config, ":agentName", agentName) + config = strings.ReplaceAll(config, ":project", testProject) + + return config +} + func testBigQuerySLO(name string) string { var serviceName = name + "-tf-service" var agentName = name + "-tf-agent"