e2e for wrangler-groupBy

pom.xml

@@ -547,7 +547,7 @@
         <dependency>
           <groupId>io.cdap.tests.e2e</groupId>
           <artifactId>cdap-e2e-framework</artifactId>
-          <version>0.3.0-SNAPSHOT</version>
+          <version>0.4.0-SNAPSHOT</version>
           <scope>test</scope>
         </dependency>
 

wrangler-transform/src/e2e-test/features/Wrangler/Runtime.feature

@@ -0,0 +1,43 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler
+Feature:  Wrangler - Run Time scenarios
+
+  @BQ_SOURCE_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using wrangler and groupBy directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_GroupBy"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_GroupBy"

wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -40,8 +40,9 @@ public class TestSetupHooks {
   @Before(order = 1, value = "@BQ_SOURCE_CSV_TEST")
   public static void createTempSourceBQTable() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFileCsv"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileCsv"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileCsv"));
   }
+
   @Before(order = 1, value = "@BQ_SINK_TEST")
   public static void setTempTargetBQTableName() {
     String bqTargetTableName = "E2E_TARGET_" + UUID.randomUUID().toString().replaceAll("-", "_");
@@ -54,7 +55,8 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce
     String bqTargetTableName = PluginPropertyUtils.pluginProp("bqTargetTable");
     try {
       BigQueryClient.dropBqQuery(bqTargetTableName);
-      BeforeActions.scenario.write("BQ Target table - " + bqTargetTableName + " deleted successfully");
+      BeforeActions.scenario.write(
+          "BQ Target table - " + bqTargetTableName + " deleted successfully");
       PluginPropertyUtils.removePluginProp("bqTargetTable");
     } catch (BigQueryException e) {
       if (e.getMessage().contains("Not found: Table")) {
@@ -71,22 +73,25 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce
   @Before(order = 1, value = "@BQ_SOURCE_FXDLEN_TEST")
   public static void createTempSourceBQTableFxdLen() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileFxdLen"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileFxdLen"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileFxdLen"));
   }
+
   @Before(order = 1, value = "@BQ_SOURCE_HL7_TEST")
   public static void createTempSourceBQTableHl7() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileHl7"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileHl7"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileHl7"));
   }
+
   @Before(order = 1, value = "@BQ_SOURCE_TS_TEST")
   public static void createTempSourceBQTableTimestamp() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileTimestamp"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileTimestamp"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileTimestamp"));
   }
+
   @Before(order = 1, value = "@BQ_SOURCE_DATETIME_TEST")
   public static void createTempSourceBQTableDateTime() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileDatetime"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileDatetime"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileDatetime"));
   }
 
   @After(order = 1, value = "@BQ_SOURCE_TEST")
@@ -95,37 +100,50 @@ public static void deleteTempSourceBQTable() throws IOException, InterruptedExce
     BigQueryClient.dropBqQuery(bqSourceTable);
     BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
     PluginPropertyUtils.removePluginProp("bqSourceTable");
+
+  }
+
+  @Before(order = 1, value = "@BQ_SOURCE_TEST")
+  public static void createTempSourceBQTableGroupBy() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFile"),
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFile"));
   }
 
-  private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile, String bqInsertDataQueryFile)
-    throws IOException, InterruptedException {
-    String bqSourceTable = "E2E_SOURCE_" + UUID.randomUUID().toString().substring(0, 5).replaceAll("-",
-                                                                                                   "_");
+  private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile,
+      String bqInsertDataQueryFile)
+      throws IOException, InterruptedException {
+    String bqSourceTable =
+        "E2E_SOURCE_" + UUID.randomUUID().toString().substring(0, 5).replaceAll("-",
+            "_");
 
     String createTableQuery = StringUtils.EMPTY;
     try {
       createTableQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource
-        ("/" + bqCreateTableQueryFile).toURI()))
-        , StandardCharsets.UTF_8);
-      createTableQuery = createTableQuery.replace("DATASET", PluginPropertyUtils.pluginProp("dataset"))
-        .replace("TABLE_NAME", bqSourceTable);
+          ("/" + bqCreateTableQueryFile).toURI()))
+          , StandardCharsets.UTF_8);
+      createTableQuery = createTableQuery.replace("DATASET",
+              PluginPropertyUtils.pluginProp("dataset"))
+          .replace("TABLE_NAME", bqSourceTable);
     } catch (Exception e) {
-      BeforeActions.scenario.write("Exception in reading " + bqCreateTableQueryFile + " - " + e.getMessage());
+      BeforeActions.scenario.write(
+          "Exception in reading " + bqCreateTableQueryFile + " - " + e.getMessage());
       Assert.fail("Exception in BigQuery testdata prerequisite setup " +
-                    "- error in reading create table query file " + e.getMessage());
+          "- error in reading create table query file " + e.getMessage());
     }
 
     String insertDataQuery = StringUtils.EMPTY;
     try {
       insertDataQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource
-        ("/" + bqInsertDataQueryFile).toURI()))
-        , StandardCharsets.UTF_8);
-      insertDataQuery = insertDataQuery.replace("DATASET", PluginPropertyUtils.pluginProp("dataset"))
-        .replace("TABLE_NAME", bqSourceTable);
+          ("/" + bqInsertDataQueryFile).toURI()))
+          , StandardCharsets.UTF_8);
+      insertDataQuery = insertDataQuery.replace("DATASET",
+              PluginPropertyUtils.pluginProp("dataset"))
+          .replace("TABLE_NAME", bqSourceTable);
     } catch (Exception e) {
-      BeforeActions.scenario.write("Exception in reading " + bqInsertDataQueryFile + " - " + e.getMessage());
+      BeforeActions.scenario.write(
+          "Exception in reading " + bqInsertDataQueryFile + " - " + e.getMessage());
       Assert.fail("Exception in BigQuery testdata prerequisite setup " +
-                    "- error in reading insert data query file " + e.getMessage());
+          "- error in reading insert data query file " + e.getMessage());
     }
     BigQueryClient.getSoleQueryResult(createTableQuery);
     try {

wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_wrangler_GroupBy

@@ -0,0 +1,5 @@
+{"city":"San Jose","cityFirst":"San Jose","firstname":"DOUGLAS","id":"1","lastname":"Williams","state":"CA","zipcode":923564293}
+{"city":"Houston","cityFirst":"Houston","firstname":"DAVID","id":"2","lastname":"Johnson","state":"TX","zipcode":1738378970}
+{"city":"Manhattan","cityFirst":"Manhattan","firstname":"HUGH","id":"3","lastname":"Jackman","state":"NY","zipcode":-1863622247}
+{"city":"San Diego","cityFirst":"San Diego","firstname":"FRANK","id":"5","lastname":"Underwood","state":"CA","zipcode":-1317090526}
+{"city":"New York","cityFirst":"New York","firstname":"SARTHAK","id":"7","lastname":"Dash","state":"NY","zipcode":-1949601773}

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQuery.txt

@@ -0,0 +1,2 @@
+create table `DATASET.TABLE_NAME` (id STRING, firstname STRING, lastname STRING, streetAddress STRING,
+city STRING, state STRING, zipcode BIGINT, phoneNumber BIGINT)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQuery.txt

@@ -0,0 +1,10 @@
+INSERT INTO DATASET.TABLE_NAME (id, firstname, lastname, streetAddress, city, state, zipcode, phoneNumber)
+VALUES
+('5', 'Frank', 'Underwood', '1609 Far St.', 'San Diego', 'CA', 2977876770, 19061512345),
+('1', 'Douglas', 'Williams', '1 Vista Montana', 'San Jose', 'CA', 9513498885, 35834612345),
+('4', 'Walter', 'White', '3828 Piermont Dr', 'Orlando', 'FL', 7349864532, 7829812345),
+('3', 'Hugh', 'Jackman', '5, Cool Way', 'Manhattan', 'NY', 6726312345, 1695412345),
+('7', 'Sarthak', 'Dash', '123 Far St.', 'New York', 'NY', 2345365523, 1324812345),
+('6', 'Serena', 'Woods', '123 Far St.', 'Las Vegas', 'NV', 4533456734, 78919612345),
+('2', 'David', 'Johnson', '3 Baypointe Parkway', 'Houston', 'TX', 1738378970, 1451412345),
+('8', 'Rahul', 'Dash', '22 MG Road.', 'Bangalore', 'KA',NULL, 94864612345);

wrangler-transform/src/e2e-test/resources/pluginParameters.properties

@@ -5,6 +5,7 @@ Directive_parse_Timestamp=testData/Wrangler/parse_timestamp_wrangle-cdap-data-pi
 Directive_parse_Datetime=testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json
 Directive_parse_csv=testData/Wrangler\
   /parse_csv_wrangle-cdap-data-pipeline.json
+Directive_GroupBy=testData/Wrangler/BQ2BQwithWrnglerNGrpby-cdap-data-pipeline (1).json
 bqSourceTable=dummy
 sourcePath=example/hello.csv
 gcsSourceBucket=dummy
@@ -20,12 +21,15 @@ CreateBQDataQueryFileDatetime=BQtesdata/BigQuery/BigQueryCreateTableQueryDatetim
 InsertBQDataQueryFileDatetime=BQtesdata/BigQuery/BigQueryInsertDataQueryDatetime.txt
 CreateBQTableQueryFileCsv=BQtesdata/BigQuery/BigQueryCreateTableQueryCsv.txt
 InsertBQDataQueryFileCsv=BQtesdata/BigQuery/BigQueryInsertDataQueryCsv.txt
+CreateBQTableQueryFile=BQtesdata/BigQuery/BigQueryCreateTableQuery.txt
+InsertBQDataQueryFile=BQtesdata/BigQuery/BigQueryInsertDataQuery.txt
 
 #bq properties
 projectId=cdf-athena
 dataset=test_automation
 dataset2=Wrangler
 #expectedBQFiles
+ExpectedDirective_GroupBy=BQValidationExpectedFiles/Directive_wrangler_GroupBy
 
 ExpectedDirective_parse_FixedLength=BQValidationExpectedFiles/Directive_parse_fixedlength
 ExpectedDirective_parse_hl7=BQValidationExpectedFiles/Directive_parse_hl7