Consider feature filter in model validation (#1258)

Signed-off-by: Kaituo Li <kaituo@amazon.com>
opensearch-project · Jul 8, 2024 · 63dacaa · 63dacaa
1 parent c059eff
commit 63dacaa
Show file tree

Hide file tree

Showing 32 changed files with 2,479 additions and 568 deletions.
diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml
@@ -52,19 +52,14 @@ jobs:
       - name: Multi Nodes Integration Testing
         run: |
           ./gradlew integTest -PnumNodes=3
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./build/reports/jacoco/test/jacocoTestReport.xml
-          flags: plugin
 
   Build-ad-linux:
     needs: [Get-CI-Image-Tag, spotless]
     strategy:
       matrix:
         java: [21]
       fail-fast: false
-    name: Build and Test Anomaly detection Plugin
+    name: Build and Test Anomaly detection Plugin on Linux
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
@@ -93,11 +88,6 @@ jobs:
                                ./gradlew build -x spotlessJava &&
                                ./gradlew publishToMavenLocal &&
                                ./gradlew integTest -PnumNodes=3"
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./build/reports/jacoco/test/jacocoTestReport.xml
-          flags: plugin
 
   Build-ad-macos:
     needs: spotless
@@ -106,7 +96,7 @@ jobs:
         java: [21]
       fail-fast: false
 
-    name: Build and Test Anomaly detection Plugin
+    name: Build and Test Anomaly detection Plugin on MacOS
     runs-on: macos-13
     env:
       JENKINS_URL: build.ci.opensearch.org
@@ -127,14 +117,17 @@ jobs:
       - name: Build and Run Tests
         run: |
           ./gradlew build -x spotlessJava
+      # coverage.gradle is only applied in single local node test
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          file: ./build/reports/jacoco/test/jacocoTestReport.xml
+          flags: plugin
       - name: Publish to Maven Local
         run: |
           ./gradlew publishToMavenLocal
       - name: Multi Nodes Integration Testing
         run: |
           ./gradlew integTest -PnumNodes=3
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./build/reports/jacoco/test/jacocoTestReport.xml
-          flags: plugin
diff --git a/src/main/java/org/opensearch/ad/constant/ADCommonMessages.java b/src/main/java/org/opensearch/ad/constant/ADCommonMessages.java
@@ -25,7 +25,6 @@ public class ADCommonMessages {
     public static String DETECTOR_IS_RUNNING = "Detector is already running";
     public static String DETECTOR_MISSING = "Detector is missing";
     public static String AD_TASK_ACTION_MISSING = "AD task action is missing";
-    public static final String INDEX_NOT_FOUND = "index does not exist";
     public static final String UNSUPPORTED_PROFILE_TYPE = "Unsupported profile types";
 
     public static final String REQUEST_THROTTLED_MSG = "Request throttled. Please try again later.";

diff --git a/src/main/java/org/opensearch/ad/rest/handler/ADModelValidationActionHandler.java b/src/main/java/org/opensearch/ad/rest/handler/ADModelValidationActionHandler.java
@@ -17,6 +17,7 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.timeseries.AnalysisType;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
+import org.opensearch.timeseries.model.ValidationIssueType;
 import org.opensearch.timeseries.rest.handler.ModelValidationActionHandler;
 import org.opensearch.timeseries.transport.ValidateConfigResponse;
 import org.opensearch.timeseries.util.SecurityClientUtil;
@@ -50,7 +51,8 @@ public ADModelValidationActionHandler(
             clock,
             settings,
             user,
-            AnalysisType.AD
+            AnalysisType.AD,
+            ValidationIssueType.DETECTION_INTERVAL
         );
     }
 

diff --git a/src/main/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportAction.java b/src/main/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportAction.java
@@ -28,6 +28,7 @@
 import org.opensearch.rest.RestRequest;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
 import org.opensearch.timeseries.model.Config;
+import org.opensearch.timeseries.model.ValidationAspect;
 import org.opensearch.timeseries.rest.handler.Processor;
 import org.opensearch.timeseries.transport.BaseValidateConfigTransportAction;
 import org.opensearch.timeseries.transport.ValidateConfigRequest;
@@ -61,7 +62,8 @@ public ValidateAnomalyDetectorTransportAction(
             actionFilters,
             transportService,
             searchFeatureDao,
-            AD_FILTER_BY_BACKEND_ROLES
+            AD_FILTER_BY_BACKEND_ROLES,
+            ValidationAspect.DETECTOR
         );
     }
 

diff --git a/src/main/java/org/opensearch/forecast/rest/handler/ForecastModelValidationActionHandler.java b/src/main/java/org/opensearch/forecast/rest/handler/ForecastModelValidationActionHandler.java
@@ -17,6 +17,7 @@
 import org.opensearch.forecast.model.Forecaster;
 import org.opensearch.timeseries.AnalysisType;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
+import org.opensearch.timeseries.model.ValidationIssueType;
 import org.opensearch.timeseries.rest.handler.ModelValidationActionHandler;
 import org.opensearch.timeseries.transport.ValidateConfigResponse;
 import org.opensearch.timeseries.util.SecurityClientUtil;
@@ -50,7 +51,8 @@ public ForecastModelValidationActionHandler(
             clock,
             settings,
             user,
-            AnalysisType.FORECAST
+            AnalysisType.FORECAST,
+            ValidationIssueType.FORECAST_INTERVAL
         );
     }
 

diff --git a/src/main/java/org/opensearch/forecast/transport/ValidateForecasterTransportAction.java b/src/main/java/org/opensearch/forecast/transport/ValidateForecasterTransportAction.java
@@ -22,6 +22,7 @@
 import org.opensearch.rest.RestRequest;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
 import org.opensearch.timeseries.model.Config;
+import org.opensearch.timeseries.model.ValidationAspect;
 import org.opensearch.timeseries.rest.handler.Processor;
 import org.opensearch.timeseries.transport.BaseValidateConfigTransportAction;
 import org.opensearch.timeseries.transport.ValidateConfigRequest;
@@ -55,7 +56,8 @@ public ValidateForecasterTransportAction(
             actionFilters,
             transportService,
             searchFeatureDao,
-            FORECAST_FILTER_BY_BACKEND_ROLES
+            FORECAST_FILTER_BY_BACKEND_ROLES,
+            ValidationAspect.FORECASTER
         );
     }
 

diff --git a/src/main/java/org/opensearch/timeseries/constant/CommonMessages.java b/src/main/java/org/opensearch/timeseries/constant/CommonMessages.java
@@ -54,9 +54,9 @@ public static String getTooManyCategoricalFieldErr(int limit) {
     public static String TIME_FIELD_NOT_ENOUGH_HISTORICAL_DATA =
         "There isn't enough historical data found with current timefield selected.";
     public static String CATEGORY_FIELD_TOO_SPARSE =
-        "Data is most likely too sparse with the given category fields. Consider revising category field/s or ingesting more data ";
+        "Data is most likely too sparse with the given category fields. Consider revising category field/s or ingesting more data.";
     public static String WINDOW_DELAY_REC =
-        "Latest seen data point is at least %d minutes ago, consider changing window delay to at least %d minutes.";
+        "Latest seen data point is at least %d minutes ago. Consider changing window delay to at least %d minutes.";
     public static String INTERVAL_REC = "The selected interval might collect sparse data. Consider changing interval length to: ";
     public static String RAW_DATA_TOO_SPARSE =
         "Source index data is potentially too sparse for model training. Consider changing interval length or ingesting more data";
@@ -65,13 +65,14 @@ public static String getTooManyCategoricalFieldErr(int limit) {
     public static String CATEGORY_FIELD_NO_DATA =
         "No entity was found with the given categorical fields. Consider revising category field/s or ingesting more data";
     public static String FEATURE_QUERY_TOO_SPARSE =
-        "Data is most likely too sparse when given feature queries are applied. Consider revising feature queries.";
+        "Data is most likely too sparse when given feature queries are applied. Consider revising feature queries";
     public static String TIMEOUT_ON_INTERVAL_REC = "Timed out getting interval recommendation";
     public static final String NOT_EXISTENT_VALIDATION_TYPE = "The given validation type doesn't exist";
     public static final String NOT_EXISTENT_SUGGEST_TYPE = "The given suggest type doesn't exist";
     public static final String DESCRIPTION_LENGTH_TOO_LONG = "Description length is too long. Max length is "
         + TimeSeriesSettings.MAX_DESCRIPTION_LENGTH
         + " characters.";
+    public static final String INDEX_NOT_FOUND = "index does not exist";
 
     // ======================================
     // Index message