Merge pull request #667 from priyabhatnagar25/wrangler-groupby-e2e

wrangler-groupBy e2e

wrangler-transform/src/e2e-test/features/Wrangler/Runtime.feature

@@ -0,0 +1,43 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler
+Feature:  Wrangler - Run time scenarios
+
+  @BQ_SOURCE_GRPBY_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using wrangler and groupBy directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_GroupBy"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQueryTable" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Validate "BigQuery2" plugin properties
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_GroupBy"

wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -108,6 +108,7 @@ public static void deleteTempSourceBQTable() throws IOException, InterruptedExce
     PluginPropertyUtils.removePluginProp("bqSourceTable");
   }
 
+
   @Before(order = 1, value = "@BQ_SOURCE_JSON_TEST")
   public static void createTempSourceBQTableJson() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFileJson"),
@@ -120,6 +121,12 @@ public static void createTempSourceBQTableXml() throws IOException, InterruptedE
         PluginPropertyUtils.pluginProp("InsertBQDataQueryFileXml"));
   }
 
+  @Before(order = 1, value = "@BQ_SOURCE_GRPBY_TEST")
+  public static void createTempSourceBQTableGroupBy() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFile"),
+                                   PluginPropertyUtils.pluginProp("InsertBQDataQueryFile"));
+  }
+
   @Before(order = 1, value = "@GCS_SOURCE_TEST")
   public static void createBucketWithEXCELFile() throws IOException, URISyntaxException {
     gcsSourceBucketName = createGCSBucketWithFile(PluginPropertyUtils.pluginProp("testFile"));
@@ -168,16 +175,14 @@ private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile
     String createTableQuery = StringUtils.EMPTY;
     try {
       createTableQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource
-          ("/" + bqCreateTableQueryFile).toURI()))
-          , StandardCharsets.UTF_8);
-      createTableQuery = createTableQuery.replace("DATASET",
-              PluginPropertyUtils.pluginProp("dataset"))
-          .replace("TABLE_NAME", bqSourceTable);
+        ("/" + bqCreateTableQueryFile).toURI()))
+        , StandardCharsets.UTF_8);
+      createTableQuery = createTableQuery.replace("DATASET", PluginPropertyUtils.pluginProp("dataset"))
+        .replace("TABLE_NAME", bqSourceTable);
     } catch (Exception e) {
-      BeforeActions.scenario.write(
-          "Exception in reading " + bqCreateTableQueryFile + " - " + e.getMessage());
+      BeforeActions.scenario.write("Exception in reading " + bqCreateTableQueryFile + " - " + e.getMessage());
       Assert.fail("Exception in BigQuery testdata prerequisite setup " +
-          "- error in reading create table query file " + e.getMessage());
+                    "- error in reading create table query file " + e.getMessage());
     }
 
     String insertDataQuery = StringUtils.EMPTY;

wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_wrangler_GroupBy

@@ -0,0 +1,5 @@
+{"city":"San Jose","cityFirst":"San Jose","firstname":"DOUGLAS","id":"1","lastname":"Williams","state":"CA","zipcode":923564293}
+{"city":"Houston","cityFirst":"Houston","firstname":"DAVID","id":"2","lastname":"Johnson","state":"TX","zipcode":1738378970}
+{"city":"Manhattan","cityFirst":"Manhattan","firstname":"HUGH","id":"3","lastname":"Jackman","state":"NY","zipcode":-1863622247}
+{"city":"San Diego","cityFirst":"San Diego","firstname":"FRANK","id":"5","lastname":"Underwood","state":"CA","zipcode":-1317090526}
+{"city":"New York","cityFirst":"New York","firstname":"SARTHAK","id":"7","lastname":"Dash","state":"NY","zipcode":-1949601773}

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQuery.txt

@@ -0,0 +1,2 @@
+create table `DATASET.TABLE_NAME` (id STRING, firstname STRING, lastname STRING, streetAddress STRING,
+city STRING, state STRING, zipcode BIGINT, phoneNumber BIGINT)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryDatetime.txt

@@ -1 +1 @@
-create table `DATASET.TABLE_NAME` (id STRING, create_date STRING, timestamp STRING)
+create table `DATASET.TABLE_NAME` (id BIGINT, create_date STRING, timestamp STRING)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryFxdlen.txt

@@ -1 +1 @@
-create table `DATASET.TABLE_NAME` (url STRING, fixedlength STRING)
+create table `DATASET.TABLE_NAME` (Url STRING, fixedlength STRING)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQuery.txt

@@ -0,0 +1,10 @@
+INSERT INTO DATASET.TABLE_NAME (id, firstname, lastname, streetAddress, city, state, zipcode, phoneNumber)
+VALUES
+('5', 'Frank', 'Underwood', '1609 Far St.', 'San Diego', 'CA', 2977876770, 19061512345),
+('1', 'Douglas', 'Williams', '1 Vista Montana', 'San Jose', 'CA', 9513498885, 35834612345),
+('4', 'Walter', 'White', '3828 Piermont Dr', 'Orlando', 'FL', 7349864532, 7829812345),
+('3', 'Hugh', 'Jackman', '5, Cool Way', 'Manhattan', 'NY', 6726312345, 1695412345),
+('7', 'Sarthak', 'Dash', '123 Far St.', 'New York', 'NY', 2345365523, 1324812345),
+('6', 'Serena', 'Woods', '123 Far St.', 'Las Vegas', 'NV', 4533456734, 78919612345),
+('2', 'David', 'Johnson', '3 Baypointe Parkway', 'Houston', 'TX', 1738378970, 1451412345),
+('8', 'Rahul', 'Dash', '22 MG Road.', 'Bangalore', 'KA',NULL, 94864612345);

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryDatetime.txt

@@ -1,5 +1,5 @@
 INSERT INTO DATASET.TABLE_NAME (id,create_date,timestamp)
 VALUES
-('1','2021-01-21','2006-02-18T05:03:42Z[UTC]'),
-('2','2022-02-22','2007-01-18T04:03:22Z[UTC]'),
-('3','2023-03-23','2008-07-19T08:04:22Z[UTC]');
+(1,'2021-01-21','2006-02-18T05:03:42Z[UTC]'),
+(2,'2022-02-22','2007-01-18T04:03:22Z[UTC]'),
+(3,'2023-03-23','2008-07-19T08:04:22Z[UTC]');

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryFxdlen.txt

@@ -1,4 +1,4 @@
-INSERT INTO DATASET.TABLE_NAME (url,fixedlength)
+INSERT INTO DATASET.TABLE_NAME (Url,fixedlength)
 VALUES
 ('http://example.com:80/docs/books/tutorial/index.html?name=networking#DOWNLOADING','21  10  ABCXYZ'),
 ('http://geeks.com:80/docs/chair/tutorial/index.html?name=networking#DOWNLOADING','19  13  ABCXYZ'),

wrangler-transform/src/e2e-test/resources/pluginParameters.properties

@@ -8,6 +8,7 @@ Directive_parse_xml=testData/Wrangler/parse_xmltojson_wrangler-cdap-data-pipelin
 Directive_parse_excel=testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json
 Directive_parse_csv=testData/Wrangler\
   /parse_csv_wrangle-cdap-data-pipeline.json
+Directive_GroupBy=testData/Wrangler/BQ2BQwithWrnglerNGrpby-cdap-data-pipeline (1).json
 bqSourceTable=dummy
 bqTargetTable=dummy
 sourcePath=example/hello.csv
@@ -29,12 +30,15 @@ CreateBQDataQueryFileXml=BQtesdata/BigQuery/BigQueryCreateTableQueryXml.txt
 InsertBQDataQueryFileXml=BQtesdata/BigQuery/BigQueryInsertDataQueryXml.txt
 CreateBQTableQueryFileCsv=BQtesdata/BigQuery/BigQueryCreateTableQueryCsv.txt
 InsertBQDataQueryFileCsv=BQtesdata/BigQuery/BigQueryInsertDataQueryCsv.txt
+CreateBQTableQueryFile=BQtesdata/BigQuery/BigQueryCreateTableQuery.txt
+InsertBQDataQueryFile=BQtesdata/BigQuery/BigQueryInsertDataQuery.txt
 
 #bq properties
 projectId=cdf-athena
 dataset=Wrangler_Test
 dataset2=Wrangler
 #expectedBQFiles
+ExpectedDirective_GroupBy=BQValidationExpectedFiles/Directive_wrangler_GroupBy
 
 ExpectedDirective_parse_FixedLength=BQValidationExpectedFiles/Directive_parse_fixedlength
 ExpectedDirective_parse_hl7=BQValidationExpectedFiles/Directive_parse_hl7