diff --git a/eventhandlers.go b/eventhandlers.go index 67836b4..7940145 100644 --- a/eventhandlers.go +++ b/eventhandlers.go @@ -16,8 +16,23 @@ import ( "github.com/vadasambar/datadog-service/pkg/utils" ) -const sliFile = "datadog/sli.yaml" -const apiSleep = 30 +const ( + sliFile = "datadog/sli.yaml" + defaultSleepBeforeAPIInSeconds = 30 +) + +// We have to put a min of 30s of sleep for the datadog API to reflect the data correctly +// More info: https://github.com/keptn-sandbox/datadog-service/issues/8 +var sleepBeforeAPIInSeconds int + +func init() { + var err error + sleepBeforeAPIInSeconds, err = strconv.Atoi(strings.TrimSpace(os.Getenv("SLEEP_BEFORE_API_IN_SECONDS"))) + if err != nil || sleepBeforeAPIInSeconds < defaultSleepBeforeAPIInSeconds { + logger.Infof("defaulting SLEEP_BEFORE_API_IN_SECONDS to 30s because it was set to '%v' which is less than the min allowed value of 30s", sleepBeforeAPIInSeconds) + sleepBeforeAPIInSeconds = defaultSleepBeforeAPIInSeconds + } +} // HandleGetSliTriggeredEvent handles get-sli.triggered events if SLIProvider == datadog func HandleGetSliTriggeredEvent(myKeptn *keptnv2.Keptn, incomingEvent cloudevents.Event, data *keptnv2.GetSLITriggeredEventData) error { @@ -29,7 +44,7 @@ func HandleGetSliTriggeredEvent(myKeptn *keptnv2.Keptn, incomingEvent cloudevent // Step 1 - Do we need to do something? // Lets make sure we are only processing an event that really belongs to our SLI Provider - if data.GetSLI.SLIProvider != "datadog-service" { + if data.GetSLI.SLIProvider != "datadog" { logger.Infof("Not handling get-sli event as it is meant for %s", data.GetSLI.SLIProvider) return nil } @@ -108,8 +123,8 @@ func HandleGetSliTriggeredEvent(myKeptn *keptnv2.Keptn, incomingEvent cloudevent // Pulling the data from Datadog api immediately gives incorrect data in api response // we have to wait for some time for the correct data to be reflected in the api response // TODO: Find a better way around the sleep time for datadog api - logger.Debugf("waiting for %vs so that the metrics data is reflected correctly in the api", apiSleep) - time.Sleep(time.Second * apiSleep) + logger.Debugf("waiting for %vs so that the metrics data is reflected correctly in the api", sleepBeforeAPIInSeconds) + time.Sleep(time.Second * time.Duration(sleepBeforeAPIInSeconds)) query := replaceQueryParameters(data, sliConfig[indicatorName], start, end) logger.Debugf("actual query sent to datadog: %v, from: %v, to: %v", query, start.Unix(), end.Unix()) diff --git a/helm/templates/secret.yaml b/helm/templates/secret.yaml index 6b16104..e4d576c 100644 --- a/helm/templates/secret.yaml +++ b/helm/templates/secret.yaml @@ -9,7 +9,7 @@ metadata: type: Opaque data: DD_API_KEY: {{ required "A valid DD_API_KEY is required to connect to the Datadog API" .Values.datadogservice.ddApikey | b64enc | quote }} - DD_APP_KEY: {{ required "A valid DD_API_KEY is required to connect to the Datadog API" .Values.datadogservice.ddAppKey | b64enc | quote }} - DD_SITE: {{ required "A valid DD_API_KEY is required to connect to the Datadog API" .Values.datadogservice.ddSite | b64enc | quote }} + DD_APP_KEY: {{ required "A valid DD_APP_KEY is required to connect to the Datadog API" .Values.datadogservice.ddAppKey | b64enc | quote }} + DD_SITE: {{ required "A valid DD_SITE is required to connect to the Datadog API" .Values.datadogservice.ddSite | b64enc | quote }} {{- end -}} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 486d456..4904741 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -14,7 +14,7 @@ datadogservice: image: repository: ghcr.io/keptn-sandbox/datadog-service # Container Image Name pullPolicy: IfNotPresent # Kubernetes Image Pull Policy - tag: 0.1.0 # Container Tag + tag: 0.2.0 # Container Tag service: enabled: true # Creates a Kubernetes Service for the datadog-service diff --git a/main.go b/main.go index 63d69ba..a0ee27f 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,9 @@ import ( var keptnOptions = keptn.KeptnOpts{} -const envVarLogLevel = "LOG_LEVEL" +const ( + envVarLogLevel = "LOG_LEVEL" +) type envConfig struct { // Port on which to listen for cloudevents diff --git a/resources/sli.yaml b/resources/sli.yaml new file mode 100644 index 0000000..de313e2 --- /dev/null +++ b/resources/sli.yaml @@ -0,0 +1,9 @@ +--- +spec_version: '1.0' +indicators: + # http_response_time_seconds_main_page_sum: sum(rate(http_server_request_duration_seconds_sum{method="GET",route="/",status_code="200",job="$SERVICE-$PROJECT-$STAGE"}[$DURATION_SECONDS])/rate(http_server_request_duration_seconds_count{method="GET",route="/",status_code="200",job="$SERVICE-$PROJECT-$STAGE"}[$DURATION_SECONDS])) + # failing_request: promhttp_metric_handler_requests_total{code!="200",job="$SERVICE-$PROJECT-$STAGE"} + # http_requests_total_sucess: http_requests_total{status="success"} + # go_routines: go_goroutines{job="$SERVICE-$PROJECT-$STAGE"} + # request_throughput: sum(rate(http_requests_total{status="success"}[2m])) + system_load: avg:system.load.1{*}.rollup(avg, $DURATION) \ No newline at end of file diff --git a/resources/slo.yaml b/resources/slo.yaml new file mode 100644 index 0000000..269a8b5 --- /dev/null +++ b/resources/slo.yaml @@ -0,0 +1,23 @@ +--- +spec_version: '0.1.0' +comparison: + compare_with: "single_result" + include_result_with_score: "pass" + aggregate_function: avg +objectives: + - sli: system_load + displayName: "System Load" + pass: + - criteria: + - "<0.6" + warning: + - criteria: + - ">=0.6" + - "<0.7" + fail: + - criteria: + - ">0.7" + +total_score: + pass: "100%" + warning: "50%" \ No newline at end of file diff --git a/test-events/evaluation.triggered.json b/test-events/evaluation.triggered.json index e009eb6..e458cf8 100644 --- a/test-events/evaluation.triggered.json +++ b/test-events/evaluation.triggered.json @@ -9,10 +9,10 @@ }, "labels": null, "message": "", - "project": "sockshop", + "project": "podtatohead", "result": "", - "service": "carts", - "stage": "staging", + "service": "helloservice", + "stage": "hardening", "status": "", "test": { "end": "", diff --git a/test-events/get-sli.triggered.json b/test-events/get-sli.triggered.json index 0e6586e..9f5ba14 100644 --- a/test-events/get-sli.triggered.json +++ b/test-events/get-sli.triggered.json @@ -4,18 +4,17 @@ "customFilters": [], "end": "2021-01-15T15:09:45.000Z", "indicators": [ - "response_time_p95", - "some_other_metric" + "system_load" ], "sliProvider": "datadog-service", "start": "2021-01-15T15:04:45.000Z" }, "labels": null, "message": "", - "project": "sockshop", + "project": "podtatohead", "result": "", - "service": "carts", - "stage": "staging", + "service": "helloservice", + "stage": "hardening", "status": "" }, "id": "409539ae-c0b9-436e-abc6-c257292e28ff",