diff --git a/README.md b/README.md index f67b1f5ac..687383d43 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ Check out the [contributions guide](https://docs.elementary-data.com/general/con + @@ -181,6 +182,9 @@ Check out the [contributions guide](https://docs.elementary-data.com/general/con + + + diff --git a/docs/guides/anomaly-detection-configuration/fail_on_zero.mdx b/docs/guides/anomaly-detection-configuration/fail_on_zero.mdx new file mode 100644 index 000000000..6c99a99aa --- /dev/null +++ b/docs/guides/anomaly-detection-configuration/fail_on_zero.mdx @@ -0,0 +1,37 @@ +--- +title: "fail_on_zero" +sidebarTitle: "fail_on_zero" +--- + +`fail_on_zero: true/false` + +Elementary anomaly detection tests will fail if there is a zero metric value within the detection period. +If undefined, default is false. + +- _Default: false_ +- _Relevant tests: All anomaly detection tests_ + + + +```yml test +models: + - name: this_is_a_model + tests: + - elementary.volume_anomalies: + fail_on_zero: true +``` + +```yml model +models: + - name: this_is_a_model + config: + elementary: + fail_on_zero: true +``` + +```yml dbt_project.yml +vars: + fail_on_zero: true +``` + + diff --git a/docs/guides/anomaly-detection-configuration/ignore_small_changes.mdx b/docs/guides/anomaly-detection-configuration/ignore_small_changes.mdx new file mode 100644 index 000000000..eb51b14d9 --- /dev/null +++ b/docs/guides/anomaly-detection-configuration/ignore_small_changes.mdx @@ -0,0 +1,64 @@ +--- +title: "ignore_small_changes" +sidebarTitle: "ignore_small_changes" +--- + +``` +ignore_small_changes: + spike_failure_percent_threshold: [int] + drop_failure_percent_threshold: [int] +``` + +If defined, an anomaly test will fail only if all the following conditions hold: + +- The z-score of the metric within the detection period is anomoulous +- One of the following holds: + - The metric within the detection period is higher than `spike_failure_percent_threshold` percentages of the mean value in the training period, if defined. + - The metric within the detection period is lower than `drop_failure_percent_threshold` percentages of the mean value in the training period, if defined + +Those settings can help to deal with situations where your metrics are stable and small changes causes to high z-scores, and therefore to anomaly. + +If undefined, default is null for both spike and drop. + +- _Default: none_ +- _Relevant tests: All anomaly detection tests_ + + + +```yml test +models: + - name: this_is_a_model + tests: + ignore_small_changes: + spike_failure_percent_threshold: 2 + drop_failure_percent_threshold: 50 +``` + +```yml model +models: + - name: this_is_a_model + config: + elementary: + ignore_small_changes: + spike_failure_percent_threshold: 2 +``` + +```yml source +sources: + - name: my_non_dbt_tables + schema: raw + tables: + - name: source_table + meta: + elementary: + ignore_small_changes: + drop_failure_percent_threshold: 50 +``` + +```yml dbt_project.yml +vars: + ignore_small_changes: + spike_failure_percent_threshold: 10 +``` + + diff --git a/docs/guides/anomaly-detection-tests/volume-anomalies.mdx b/docs/guides/anomaly-detection-tests/volume-anomalies.mdx index 236743cd3..69ccd0484 100644 --- a/docs/guides/anomaly-detection-tests/volume-anomalies.mdx +++ b/docs/guides/anomaly-detection-tests/volume-anomalies.mdx @@ -36,6 +36,10 @@ No mandatory configuration, however it is highly recommended to configure a `tim       period: [hour | day | week | month]       count: int     seasonality: day_of_week +     fail_on_zero: [true | false] +     ignore_small_changes: +       spike_failure_percent_threshold: int +       drop_failure_percent_threshold: int diff --git a/docs/mint.json b/docs/mint.json index 93fd43324..11ca575f1 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -147,7 +147,9 @@ "guides/anomaly-detection-configuration/exclude_regexp", "guides/anomaly-detection-configuration/dimensions", "guides/anomaly-detection-configuration/event_timestamp_column", - "guides/anomaly-detection-configuration/update_timestamp_column" + "guides/anomaly-detection-configuration/update_timestamp_column", + "guides/anomaly-detection-configuration/ignore_small_changes", + "guides/anomaly-detection-configuration/fail_on_zero" ] } ] diff --git a/elementary/monitor/data_monitoring/report/index.html b/elementary/monitor/data_monitoring/report/index.html index ac153cd56..358589e90 100644 --- a/elementary/monitor/data_monitoring/report/index.html +++ b/elementary/monitor/data_monitoring/report/index.html @@ -25,7 +25,7 @@