From e4fad2071adb96cf04e1965fef6a99b6b1a97063 Mon Sep 17 00:00:00 2001
From: priyabhatnagar <priya.bhatnagar@cloudsufi.com>
Date: Wed, 21 Jun 2023 22:25:25 +0530
Subject: [PATCH] wrangler e2e tests

---
 pom.xml                                       |   6 +-
 wrangler-transform/pom.xml                    |   2 +-
 .../features/Wrangler/ParseAsCsv.feature      |   2 +-
 .../features/Wrangler/ParseAsExcel.feature    |  42 ++
 .../features/Wrangler/ParseAsJson.feature     |  43 +++
 .../Wrangler/ParseAsXmlToJson.feature         |  43 +++
 .../common/stepsdesign/TestSetupHooks.java    | 113 ++++--
 .../Directive_parse_excel                     |   2 +
 .../Directive_parse_json                      |   3 +
 .../Directive_parse_xmltojson                 |   6 +
 .../BigQuery/BigQueryCreateTableQueryXml.txt  |   1 +
 .../BigQuery/BigQueryInsertDataQueryXml.txt   |   5 +
 .../BigQueryInsertDataQueryparsejson.txt      |   6 +
 .../BigQuery/BigQuerycreateTableQueryjson.txt |   1 +
 .../resources/BQtesdata/BigQuery/test1.xlsx   | Bin 0 -> 5228 bytes
 .../resources/pluginParameters.properties     |  11 +
 ...xcel_wrangler_copy-cdap-data-pipeline.json | 186 +++++++++
 ...e_json_Wrangle-cdap-data-pipeline (1).json | 180 +++++++++
 ..._xmltojson_wrangle-cdap-data-pipeline.json | 362 ++++++++++++++++++
 19 files changed, 986 insertions(+), 28 deletions(-)
 create mode 100644 wrangler-transform/src/e2e-test/features/Wrangler/ParseAsExcel.feature
 create mode 100644 wrangler-transform/src/e2e-test/features/Wrangler/ParseAsJson.feature
 create mode 100644 wrangler-transform/src/e2e-test/features/Wrangler/ParseAsXmlToJson.feature
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_excel
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_json
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_xmltojson
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryXml.txt
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryXml.txt
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryparsejson.txt
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQuerycreateTableQueryjson.txt
 create mode 100644 wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/test1.xlsx
 create mode 100644 wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json
 create mode 100644 wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_json_Wrangle-cdap-data-pipeline (1).json
 create mode 100644 wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_xmltojson_wrangle-cdap-data-pipeline.json

diff --git a/pom.xml b/pom.xml
index 75561b6e0..2fda3fbe9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
 
   <groupId>io.cdap.wrangler</groupId>
   <artifactId>wrangler</artifactId>
-  <version>4.11.0-SNAPSHOT</version>
+  <version>4.10.0-SNAPSHOT</version>
   <name>Wrangler</name>
   <packaging>pom</packaging>
   <description>An interactive tool for data cleansing and transformation.</description>
@@ -492,7 +492,7 @@
               <execution>
                 <goals>
                   <goal>integration-test</goal>
-<!--                  <goal>verify</goal>-->
+                  <goal>verify</goal>
                 </goals>
               </execution>
             </executions>
@@ -547,7 +547,7 @@
         <dependency>
           <groupId>io.cdap.tests.e2e</groupId>
           <artifactId>cdap-e2e-framework</artifactId>
-          <version>0.3.0-SNAPSHOT</version>
+          <version>0.4.0-SNAPSHOT</version>
           <scope>test</scope>
         </dependency>
 
diff --git a/wrangler-transform/pom.xml b/wrangler-transform/pom.xml
index 46fd633c2..b09712c0d 100644
--- a/wrangler-transform/pom.xml
+++ b/wrangler-transform/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>wrangler</artifactId>
     <groupId>io.cdap.wrangler</groupId>
-    <version>4.11.0-SNAPSHOT</version>
+    <version>4.10.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
diff --git a/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsCsv.feature b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsCsv.feature
index fa59cb54c..7cb04063d 100644
--- a/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsCsv.feature
+++ b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsCsv.feature
@@ -13,7 +13,7 @@
 # the License.
 
 @Wrangler
-Feature:  Wrangler - Run time scenarios
+Feature:  Wrangler - Run time scenarios for parse csv
 
   @BQ_SOURCE_CSV_TEST @BQ_SINK_TEST
   Scenario: To verify User is able to run a pipeline using parse csv directive
diff --git a/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsExcel.feature b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsExcel.feature
new file mode 100644
index 000000000..9d5a485c3
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsExcel.feature
@@ -0,0 +1,42 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler_Required
+Feature:  Parse as Excel
+
+  @GCS_SOURCE_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using parse Excel directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_parse_excel"
+    Then Navigate to the properties page of plugin: "GCSFile"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "path" with value: "gcsSourceBucket"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_excel"
diff --git a/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsJson.feature b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsJson.feature
new file mode 100644
index 000000000..2e85bee48
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsJson.feature
@@ -0,0 +1,43 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler
+Feature:  parse as Json
+
+  @BQ_SOURCE_JSON_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using parse Json directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_parse_json"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_json"
diff --git a/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsXmlToJson.feature b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsXmlToJson.feature
new file mode 100644
index 000000000..77485db35
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsXmlToJson.feature
@@ -0,0 +1,43 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler
+Feature:  parse as XmlToJson
+
+  @BQ_SOURCE_XML_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using parse XmlToJson directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_parse_xml"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_xml"
diff --git a/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
index 0243dc4ed..e87cc5298 100644
--- a/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
+++ b/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
@@ -17,8 +17,11 @@
 package io.cdap.plugin.common.stepsdesign;
 
 import com.google.cloud.bigquery.BigQueryException;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.StorageException;
 import io.cdap.e2e.utils.BigQueryClient;
 import io.cdap.e2e.utils.PluginPropertyUtils;
+import io.cdap.e2e.utils.StorageClient;
 import io.cucumber.java.After;
 import io.cucumber.java.Before;
 import org.apache.commons.lang3.StringUtils;
@@ -26,6 +29,7 @@
 import stepsdesign.BeforeActions;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -37,11 +41,14 @@
  */
 public class TestSetupHooks {
 
+  public static String gcsSourceBucketName = StringUtils.EMPTY;
+
   @Before(order = 1, value = "@BQ_SOURCE_CSV_TEST")
   public static void createTempSourceBQTable() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFileCsv"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileCsv"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileCsv"));
   }
+
   @Before(order = 1, value = "@BQ_SINK_TEST")
   public static void setTempTargetBQTableName() {
     String bqTargetTableName = "E2E_TARGET_" + UUID.randomUUID().toString().replaceAll("-", "_");
@@ -54,7 +61,8 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce
     String bqTargetTableName = PluginPropertyUtils.pluginProp("bqTargetTable");
     try {
       BigQueryClient.dropBqQuery(bqTargetTableName);
-      BeforeActions.scenario.write("BQ Target table - " + bqTargetTableName + " deleted successfully");
+      BeforeActions.scenario.write(
+          "BQ Target table - " + bqTargetTableName + " deleted successfully");
       PluginPropertyUtils.removePluginProp("bqTargetTable");
     } catch (BigQueryException e) {
       if (e.getMessage().contains("Not found: Table")) {
@@ -66,27 +74,30 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce
   }
 
   /**
-   * Create BigQuery table.
+   * Create BigQuery table test.
    */
   @Before(order = 1, value = "@BQ_SOURCE_FXDLEN_TEST")
   public static void createTempSourceBQTableFxdLen() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileFxdLen"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileFxdLen"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileFxdLen"));
   }
+
   @Before(order = 1, value = "@BQ_SOURCE_HL7_TEST")
   public static void createTempSourceBQTableHl7() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileHl7"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileHl7"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileHl7"));
   }
+
   @Before(order = 1, value = "@BQ_SOURCE_TS_TEST")
   public static void createTempSourceBQTableTimestamp() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileTimestamp"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileTimestamp"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileTimestamp"));
   }
+
   @Before(order = 1, value = "@BQ_SOURCE_DATETIME_TEST")
   public static void createTempSourceBQTableDateTime() throws IOException, InterruptedException {
     createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileDatetime"),
-            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileDatetime"));
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileDatetime"));
   }
 
   @After(order = 1, value = "@BQ_SOURCE_TEST")
@@ -97,35 +108,91 @@ public static void deleteTempSourceBQTable() throws IOException, InterruptedExce
     PluginPropertyUtils.removePluginProp("bqSourceTable");
   }
 
-  private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile, String bqInsertDataQueryFile)
-    throws IOException, InterruptedException {
-    String bqSourceTable = "E2E_SOURCE_" + UUID.randomUUID().toString().substring(0, 5).replaceAll("-",
-                                                                                                   "_");
+  @Before(order = 1, value = "@BQ_SOURCE_JSON_TEST")
+  public static void createTempSourceBQTableJson() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFileJson"),
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileJson"));
+  }
+
+  @Before(order = 1, value = "@BQ_SOURCE_XML_TEST")
+  public static void createTempSourceBQTableXml() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileXml"),
+        PluginPropertyUtils.pluginProp("InsertBQDataQueryFileXml"));
+  }
+
+  @Before(order = 1, value = "@GCS_SOURCE_TEST")
+  public static void createBucketWithEXCELFile() throws IOException, URISyntaxException {
+    gcsSourceBucketName = createGCSBucketWithFile(PluginPropertyUtils.pluginProp("testFile"));
+    PluginPropertyUtils.addPluginProp("gcsSourceBucket", "gs://" + gcsSourceBucketName + "/" +
+        PluginPropertyUtils.pluginProp("testFile"));
+    BeforeActions.scenario.write("GCS source bucket1 name - " + gcsSourceBucketName);
+  }
+
+  private static String createGCSBucketWithFile(String filePath)
+      throws IOException, URISyntaxException {
+    String bucketName = StorageClient.createBucket("e2e-test-" + UUID.randomUUID()).getName();
+    StorageClient.uploadObject(bucketName, filePath, filePath);
+    return bucketName;
+  }
+
+  @After(order = 1, value = "@GCS_SOURCE_TEST")
+  public static void deleteSourceBucketWithFile() {
+    deleteGCSBucket(gcsSourceBucketName);
+    gcsSourceBucketName = StringUtils.EMPTY;
+  }
+
+  private static void deleteGCSBucket(String bucketName) {
+    try {
+      for (Blob blob : StorageClient.listObjects(bucketName).iterateAll()) {
+        StorageClient.deleteObject(bucketName, blob.getName());
+      }
+      StorageClient.deleteBucket(bucketName);
+      BeforeActions.scenario.write("Deleted GCS Bucket " + bucketName);
+    } catch (StorageException | IOException e) {
+      if (e.getMessage().contains("The specified bucket does not exist")) {
+        BeforeActions.scenario.write("GCS Bucket " + bucketName + " does not exist.");
+      } else {
+        Assert.fail(e.getMessage());
+      }
+    }
+  }
+
+
+  private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile,
+      String bqInsertDataQueryFile)
+      throws IOException, InterruptedException {
+    String bqSourceTable =
+        "E2E_SOURCE_" + UUID.randomUUID().toString().substring(0, 5).replaceAll("-",
+            "_");
 
     String createTableQuery = StringUtils.EMPTY;
     try {
       createTableQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource
-        ("/" + bqCreateTableQueryFile).toURI()))
-        , StandardCharsets.UTF_8);
-      createTableQuery = createTableQuery.replace("DATASET", PluginPropertyUtils.pluginProp("dataset"))
-        .replace("TABLE_NAME", bqSourceTable);
+          ("/" + bqCreateTableQueryFile).toURI()))
+          , StandardCharsets.UTF_8);
+      createTableQuery = createTableQuery.replace("DATASET",
+              PluginPropertyUtils.pluginProp("dataset"))
+          .replace("TABLE_NAME", bqSourceTable);
     } catch (Exception e) {
-      BeforeActions.scenario.write("Exception in reading " + bqCreateTableQueryFile + " - " + e.getMessage());
+      BeforeActions.scenario.write(
+          "Exception in reading " + bqCreateTableQueryFile + " - " + e.getMessage());
       Assert.fail("Exception in BigQuery testdata prerequisite setup " +
-                    "- error in reading create table query file " + e.getMessage());
+          "- error in reading create table query file " + e.getMessage());
     }
 
     String insertDataQuery = StringUtils.EMPTY;
     try {
       insertDataQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource
-        ("/" + bqInsertDataQueryFile).toURI()))
-        , StandardCharsets.UTF_8);
-      insertDataQuery = insertDataQuery.replace("DATASET", PluginPropertyUtils.pluginProp("dataset"))
-        .replace("TABLE_NAME", bqSourceTable);
+          ("/" + bqInsertDataQueryFile).toURI()))
+          , StandardCharsets.UTF_8);
+      insertDataQuery = insertDataQuery.replace("DATASET",
+              PluginPropertyUtils.pluginProp("dataset"))
+          .replace("TABLE_NAME", bqSourceTable);
     } catch (Exception e) {
-      BeforeActions.scenario.write("Exception in reading " + bqInsertDataQueryFile + " - " + e.getMessage());
+      BeforeActions.scenario.write(
+          "Exception in reading " + bqInsertDataQueryFile + " - " + e.getMessage());
       Assert.fail("Exception in BigQuery testdata prerequisite setup " +
-                    "- error in reading insert data query file " + e.getMessage());
+          "- error in reading insert data query file " + e.getMessage());
     }
     BigQueryClient.getSoleQueryResult(createTableQuery);
     try {
diff --git a/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_excel b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_excel
new file mode 100644
index 000000000..3c3ae5154
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_excel
@@ -0,0 +1,2 @@
+{"copiedname":"very","id":0,"name":"very","phone":"8838.0","rollno":"3.0","uniquenum":"very,0"}
+{"copiedname":"hello","id":2,"name":"hell","phone":"12345.0","rollno":"1.0","uniquenum":"hello,2"}
\ No newline at end of file
diff --git a/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_json b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_json
new file mode 100644
index 000000000..b8cac65cb
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_json
@@ -0,0 +1,3 @@
+{"Body":"hello abc","copied":{"first":"Root","last":"joy"},"desc":"nick, hello abc","id":22,"json_age":"{\"json_id\":22,\"copied\":{\"first\":\"Root\",\"last\":\"joy\"},\"json_age\":1,\"json_name\":{\"first\":\"Root\",\"last\":\"joy\"},\"json_pet\":\"testing\",\"json_id_json_name\":\"22,{\\\"first\\\":\\\"Root\\\",\\\"last\\\":\\\"joy\\\"}\",\"body\":\"hello abc\",\"desc\":\"nick, hello abc\"}","json_id_json_name":"22,{\"first\":\"Root\",\"last\":\"joy\"}","json_name":{"first":"Root","last":"joy"},"json_pet":"testing"}
+{"Body":"hello def","copied":{"first":"dded","last":"share"},"desc":"hello, hello def","id":23,"json_age":"{\"json_id\":23,\"copied\":{\"first\":\"dded\",\"last\":\"share\"},\"json_age\":2,\"json_name\":{\"first\":\"dded\",\"last\":\"share\"},\"json_pet\":\"testing\",\"json_id_json_name\":\"23,{\\\"first\\\":\\\"dded\\\",\\\"last\\\":\\\"share\\\"}\",\"body\":\"hello def\",\"desc\":\"hello, hello def\"}","json_id_json_name":"23,{\"first\":\"dded\",\"last\":\"share\"}","json_name":{"first":"dded","last":"share"},"json_pet":"testing"}
+{"Body":"hello ghi","copied":{"first":"Root","last":"Joltie"},"desc":"doms, hello ghi","id":24,"json_age":"{\"json_id\":24,\"copied\":{\"first\":\"Root\",\"last\":\"Joltie\"},\"json_age\":3,\"json_name\":{\"first\":\"Root\",\"last\":\"Joltie\"},\"json_pet\":\"testing\",\"json_id_json_name\":\"24,{\\\"first\\\":\\\"Root\\\",\\\"last\\\":\\\"Joltie\\\"}\",\"body\":\"hello ghi\",\"desc\":\"doms, hello ghi\"}","json_id_json_name":"24,{\"first\":\"Root\",\"last\":\"Joltie\"}","json_name":{"first":"Root","last":"Joltie"},"json_pet":"testing"}
\ No newline at end of file
diff --git a/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_xmltojson b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_xmltojson
new file mode 100644
index 000000000..2f9d3ba92
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_xmltojson
@@ -0,0 +1,6 @@
+{"Email":"abc01@mail.com","distance":2.0,"distance2":0.3571428656578064,"email_domain":{"distance":2.0,"email_account":"abc01"},"email_porter":["abc","mail","com"],"id":"1","xmldata_note":{"body":"Dont forget me this week!","from":"Tani","heading":"Reminder","to":"Tove"}}
+{"Email":"def02@mail.com","distance":2.0,"distance2":0.3571428656578064,"email_domain":{"distance":2.0,"email_account":"def02"},"email_porter":["def","mail","com"],"id":"2","xmldata_note":{"body":"Dont forget us this holiday!","from":"joy","heading":"Reminder","to":"Tove"}}
+{"Email":"ghi03@mail.com","distance":2.0,"distance2":0.3571428656578064,"email_domain":{"distance":2.0,"email_account":"ghi03"},"email_porter":["ghi","mail","com"],"id":"3","xmldata_note":{"body":"Dont forget him this weekend!","from":"shree","heading":"Reminder","to":"Tove"}}
+{"Email":"abc01@mail.com","distance":2.0,"distance2":0.3571428656578064,"email_domain":{"distance":2.0,"email_account":"abc01"},"email_porter":["abc","mail","com"],"id":"abc","xmldata_note":{"body":"Dont forget me this week!","from":"Tani","heading":"Reminder","to":"Tove"}}
+{"Email":"def02@mail.com","distance":2.0,"distance2":0.3571428656578064,"email_domain":{"distance":2.0,"email_account":"def02"},"email_porter":["def","mail","com"],"id":"def","xmldata_note":{"body":"Dont forget us this holiday!","from":"joy","heading":"Reminder","to":"Tove"}}
+{"Email":"ghi03@mail.com","distance":2.0,"distance2":0.3571428656578064,"email_domain":{"distance":2.0,"email_account":"ghi03"},"email_porter":["ghi","mail","com"],"id":"ghi","xmldata_note":{"body":"Dont forget him this weekend!","from":"shree","heading":"Reminder","to":"Tove"}}
\ No newline at end of file
diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryXml.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryXml.txt
new file mode 100644
index 000000000..a711921e2
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryXml.txt
@@ -0,0 +1 @@
+create table `DATASET.TABLE_NAME` (email STRING, xmldata STRING)
diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryXml.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryXml.txt
new file mode 100644
index 000000000..0dc9608ce
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryXml.txt
@@ -0,0 +1,5 @@
+INSERT INTO DATASET.TABLE_NAME (email,xmldata)
+VALUES
+('abc01@mail.com','<?xml version=1.0 encoding=UTF-8?> <note> <to>Tove</to> <from>Tani</from> <heading>Reminder</heading> <body>Dont forget me this week!</body> </note>'),
+('def02@mail.com','<?xml version=1.0 encoding=UTF-8?> <note> <to>Tove</to> <from>joy</from> <heading>Reminder</heading> <body>Dont forget us this holiday!</body> </note>'),
+('ghi03@mail.com','<?xml version=1.0 encoding=UTF-8?> <note> <to>Tove</to> <from>shree</from> <heading>Reminder</heading> <body>Dont forget him this weekend!</body> </note>');
diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryparsejson.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryparsejson.txt
new file mode 100644
index 000000000..dc9fa7d17
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryparsejson.txt
@@ -0,0 +1,6 @@
+INSERT INTO DATASET.TABLE_NAME (body,json)
+VALUES
+(' hello abc', '{"id": 1, "name": {"first": "Root", "last": "joy"}, "age": 22, "pet": "nick", "height": 5.8}'),
+('hello def', '{"id": 2, "name": {"first": "dded", "last": "share"}, "age": 23, "pet": "hello", "height": 6.8}'),
+('hello ghi', '{"id": 3, "name": {"first": "Root", "last": "Joltie"}, "age": 24, "pet": "doms", "height": 7.8}');
+
diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQuerycreateTableQueryjson.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQuerycreateTableQueryjson.txt
new file mode 100644
index 000000000..be6b585ea
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQuerycreateTableQueryjson.txt
@@ -0,0 +1 @@
+create table `DATASET.TABLE_NAME` (body STRING, json STRING)
\ No newline at end of file
diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/test1.xlsx b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/test1.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..adaa5291b024baf3415f6081d3fdca5369b6bbb8
GIT binary patch
literal 5228
zcmaJ_2RNMT@?X(wbkVy+7j;Rn2vJsB(W6_PWr^NPqW6Rd5-m=m6TNp<2{zGt3!+3j
zh;~=b{a<eW_ni5j{dS*cznSlSXMXR@Z$?7}9fKTzgM$MAX%g!KZU{c=w+S5T;L6W;
zbuUV6RcjX@c)sHk$#o<$1H#FAUk_HSXBK8?VRNFbjL7K-+57Vm87V<bd3&sjf53%#
z-YnROXI6ki#W+G+KjKy+hlO8$N8g2u7r*F0wtp81@G%4?#|4j5S3c-pkU`}7xWy~u
zt?jF-45NLs*t7&lf?mLlv^@kbEW=kr^*5tQU95Go#{)1LLRB5@MVyov>Aa<c7Ij&Y
znB9An!8yM9zy?x6=yN=;S*Txa-`q|z5jBB;qAE!W^XhgFC9*n}`8>IAJvEiAu67mf
zY<_?>&_Eh0VD09k%Ic>G-hN9#FJz(|WsrH+w@Rj=f{hcFCY-j7a>C#(06^n!-E<rE
zgr@_)2Mlg+4ujeAc{)18X!SZx2@p3PabwIn@*3OGzCzcHjR(vS=l8x&RG_qvB=6aM
zm3=u&0?p%`l)Eh|jz8D%+RI=z-F}_ERWD1szs;=Wp++rV=w#>Sl!Zj<{m(4Am?;J_
z{d}eq0~<4YxdT#2@10>EsD%!*DE?fv7p6$H+)QA$;s@1%1aQT^qeUQ9)vXqTpPIzr
z^ECWUiO{eUMgpT(EH7dPZ;Ra91Q964cFgKzmpN4OgoWl8GslSySR0dVh4!VN6zEQx
z#H8%{2G~FlvHHDvn0B8=^7$t`dJ-fSA%Z3II13N&9B~GKY;wU|a)YD0Z1a`@-G)f!
zSzA`)PqKW~wZo7uf`J8tnDFeLONxoA5bx#JTN&T1v+(>Psf*V3R{h~))|#gB`024$
zOS2x>=DKG6Rtpc!8TK}_=%4g+ph<Fvm90er$<;FGf;K8CMV6b_d$tOtC0DREjGyyr
zOYGo(+;j^jm%!r|REi#u)o7WfN|4($`(q}3m7kbk<?hdS;j-g!K@I;AOqoL5Et&wM
z!LPG{kJQ*+$PibRI(NlU&Uk4jesQ~0QH5AyNt^>Y@9&sC>i2JWtAMl=jlmk0pd)Pi
z;VyVAK%<Ve?wB?*pQtLR9*|f_w-M~G^mr~i3$~1g*b39}*=DP9PxeJ@T+j<O>vn8V
zU1285Q)uf54FGUL|0m24{)HJ=H!laM>lI?Q^o-P}1jzm7N=nb95??_~l|$}m<u`(D
z$Z>5aLzL1x+87hIWZO&To~7sFhN*HTjVk+<@AOJ`-yIwK_-4(V9q64qDZp;+=|*n;
z2$~Jy*r#qX>X*(mv*bzOTu>gNF8-sHYWGFYcqH;oFeC~jlSa)~$SaK*q=d&6E)HRT
z`GZZ%fEYh(5M9eo8M34pjJ+g~juk8rv82e?`BX22v@)5}!HsyYy_&pb9fOtMBM(x#
z_fq*IZnq2_W(<f9s$BexV;tSZGIN-9fr+%vO$p<0H_ytLnKgJ95n_SOhB=HiJBZwn
zZ}>tPyf8TQz4c{;%j`?rXi4J7>}9X)#9shJ_T{x@i`X5SQa<coBxcw{V!G~1%1XA>
z9VzDzIh@D*NvurW`;zUfDt#f^S<VI9PgA`db159zA6#nXwV+UdXo=7(zyKS>8cuS1
zkm*TfrO(w2HujCe7=zcirZ$E3v7MRHkE}LcrUfRmm#iWp^cr&sf7Gf`gHqC0AVX9U
zV+Q(Z<3H!WI@2YHl29mDSjW55ri#UE4}O3~gq)q4TdpH6G+ud9{5Z%}ss)sCvn1Iz
z3Op1Fv)VJuS}pV;fO0*c4KG*@dSBEVoCn&1Vij!4=e0e{P?#G1uzOOj;a6kwp^EU&
z?*3!bRYD=dw?!eZ|L=0_2giD>r@#d#ImMDP;6k(PV&5(&z(dVVr+vPc49#02{x2OR
zc22$W5B=(q&G`<OPVGnU>|~IEjm<BOgw$Ma%WvQQNf6*M$LCq%e5BIFn<RxNimR)P
zGqM_|h84ObhP!5cT)?V|ox5I;n>YG|L%Um?-kBp{kziDU_J^9ul95(M3xiwoK3Zph
zfF2cD@S*klW3ekh%Yu0$U!f9U0qH*h`p$I{bhUv(-CX&9ecivpcB76eYzj!+xKl#x
zwq|QTQEv3$!zPQ%=oID%FdWHSDrS(&!$SU;owL5TWRCfex=#<oN0BHNDCF%6d!A-?
zu%gS+fUcDm^7J^Tb?7KHItX5Cay-X4#`vu%pD-aAC*EN}T4*ak|ALZUMshv*{t|QG
z7kp_Ej%I>e^!Yrj-T?8WH6;+VV;-`Fp24l~hYu&243N$rPp~b|gyreSYEw|+kJK5^
zQ%i6f>7)BH1S4adpP$A28P$Xv#|3}%LF6q_=bI7ad<=(VLv%NuFq;*zoQ~%Af(9Hu
z<MsZ<YT~|dZ?3&Kob;R;daCJ%4`MF_;qylbl^OxCAA007rSKsFKk<nQUZFYHZ(?!I
zanC2|d2-%)dr@n~a(;pyk*iQ%d|iW!l#%ew!v<sP_BYqbW9^r9akA}!)@NES-hJ$H
z1)B0u-@qlj4qY%tW?}IX<8$f)!pI#hOV2)1B#5SbO0d<u-R7uK)eOU_+;7mGi3Wfm
zh2U6l&bz;}J4K`B_q9cG5|hH~J0o$;pg)ZrEG}iw)XC<2(0Z!wlceSjJCEzn)sCqb
ziDpzUQ{$z)LfA_`r*BCT(Vx!ZP1G_&z9BnZc*5fw_wKMoSu}Nbu+H}@QrbucVTYlL
zTle{y?dW!g>4yTaDW$)&9b{RA2AtnM;Y+Ue=AoH(li(Xyw-@MVr6#2`m3av1<cKCH
zHiy70$;PDf3y8^&NKogu=Xc7Z$>}5u!v@^42rZ!+74r1~2u^6ZOY-ogOix+s25zFg
zkJN@7<J6V|v~r5VWYk8I=Y^&-=)DVt%S_yz##p^W);10nkWoy{kBrVRdh<QH!|$b!
z{d`%r2^4^t;LrGU%@r*!aZf#2oTtfv;w0nUsWH||C&vwKy$`t455_ad*StrxfRdW1
zKR$QwlljSfGwi2Gu}Z2xS<4%bcX?k)=sPF6qlHhDP?z!k-7&tRTnF9@KCL2QA6J2g
z+Yf@_q<*Y+pIs~#49=Vd-h6(qJqJq}r=A(Gpczb)244^nSe>Tdedd~nsR#gD(+g{r
z_LQBhf*nur-RJT9jq-=<*Y!O5>?S@vo~t0HUHLQ1IXR9qiS|hprY>D)9c_2pXBldF
zVAWv;(=PI5t~A6cE$I8gdA88Bxx*fJ&*oS8sS~!>GrG1G`3R$gExid*{DIN@RU~A3
znNAwQom(tpAHn>sKZC4a{&i7hU&3RrMAM91mt)?6$%d?hI60r?6cP=C*xeQE#Fe+Y
z+ehu6Z}oqB+yJuKA>zlr9Zzy{iaxcp1$wDQC5AWCTyW3EjK^GQ^^$oq-E;e7wpB+@
zbLeYC-)eB(*{>qV^iDly8&v?8@%|}i(EOD%{-T&S>C<0yGeOn<7u`IkP&n#T!nTM8
zHb%>ooFAigX)X?Rh$fC!Ui$OV8sb4WhmwZI_L~$BH-uw0!aL*$us&vEgtd0M<1_}H
zZ*MJ7>kMSi&fBoZl`@hRG?0x4QYrBpRO|!w`OMqm9~kBKDyw}M>#Ymf(&X>kQ$nVr
z4Nzzm8KQr3;?}5lZM!VB7$YPy$?<tB&_jqj0B1-L8ivG~&vWw&Vjno3tVpg%v(S&T
zB1%HuLPR}qznWMv?0DnDaHoni{5Q|o_V=n^%NLc*&iLNC%BJc+PP|7@`)2<~N=g22
zdSGJ)hgxd6!EK$af3bv^co{qN`vmew;$g#5yTVZ#!CO4HBxT?Lp`2*Ks!hGL_-QGp
zEH_=Nli?(WpQ{J`N|L26JeCU<Vu7+WedM-P&W*l~(lTY9X}(;cl*T>IaN2gBk0qpr
zI4p;Mh{T~)d^7r>=m+QK^;s_)NVAudb^#@)ANb!Pk~md-*Hs>`2Mjd0H%s;K(#jer
zv7ZBkA6keV(ME3t%`aMh0_HR)1PJU=>jsG&jOiqxX>omDmX}SprI`92a2`~r`+fGx
z0f^BimS)uUQc)b~?>RLI>IqAjg$5kv?8<KegF}C@YKW0~2#O;e>Bcw7;rB!Bu~feB
z+;{1E6x~&Au&qa-jC41-oV^vIIuimrns)Wx*0tXE)AlwTQurb$V+M-HP8_R^mk%~B
z50jS3ECa6=7P7Og0k8&;1=PC9V>Fp=MIb}8E?0#SjbhSIRXUo{geG#G;Cb?qgaQNI
zeu+Qu3-4T-W6p=i-!E<{PuG7=D`!F&yJquxzZK%>HbJvYCx=U+XzzR{^f<Tj9k=VQ
zOkX-Dx7lqxK4l-c#!k$g(E1s^@y?<zhm4EWEA6fiKJr^ua1q_hpr_4xie3@#GZ^T;
z;=L-;)YwOPFGi94)N(P&mNM3c4mo___dK8xMo?cfYTKR}809MP6dyW6_+amG+d^Wm
z`4VMj@WBqFVQ;*hn!cb#%Jv#J+Q~qx*VCRudhPCMX$8D)eNWZLp`>t82y94aw3K3C
zP9>*m4e+LTV0DSCW=nx4Fsi7O-ECd!Y@ZPcy9(3!xNV0GDnwKNh*RR5VKQ@ezKT*t
zyeh2is!cql(C(;I;-qCt#&S$^mz^Y{pDaR<QVFi8C`BTgO#p<z&v#EN56Au2y@l(5
zpm4+lma0n_UTK>W)PwM+!yJO0%Y0D>zQrTFhC@;`fqsrND5r>Lo`5SVYV$VHgoLz{
zfglc8?YMowcPA50?cQ<NNcv-=!?cA1qWd+yCACF4KCk)Ka9E8*?xk!VS^nu_mE_NU
zi%*zKBI+&oYV(Rzo>n6O1r@%$YZXTZxS8dw4mnNe4R3YhF*6r1i|^7xUzD914A!4m
zxQGp{Q2B${1PeI$Lq^K`r+vmF8XENm-mBLjTo{g#_iAeq{%BWj<K;`ZeS&gI+CQS3
z?55i+p1Hcg9DjLkunezFfIMtrCLd>F>Irtj$fqdagC|GvvsGxWupZSYKOffM?bsKL
zQff%abGbs|2@<<ner`?xFCo!gUf6jaP^q<d-jpHq-3k^@z5CcxR?P5Tv?v}Xd+~g7
zyS(widBPNN3qDqP7$nmLA{;x|_Ih&>ElJ|ZSIvnqvJ)0x@sr&Rk#iLy%ekH7Ss1gA
zV~)S|)USlTFbW7vL9Kch_4)f#d$lh^Ihd0h)XB}{iI+3f)#w-hdorNj&P3k4Q({w-
z*0$Pu2kC?@9b4DoC5ltRTkMN5K7HYn^%eTM(p7&cqQ$3~-<au*qh&^6t3;M|EQe<_
zsRr0J3qjY&!!Tkm{3f=%b2LyzwgfKzu|Z5M6sQxFOJtAE^#sq6{m{%_#r&S(2x-rX
zr>}gf4nwlK-!cffNrYGT(GT8?+g;=on6q596$KH`?)nm}$%tUnQMx-QyX75d-+6mF
z&t%XUBq~`<+GTWBZ2H8DiOCX7`MI8C2dj=*mDx12?U`XK^J}f|j2{RHmqj?3J?9@9
zDZfecwW;?-dPkg=G|H(`g_*X981@2M2ld`fYXgi&YxK)<W$A}K{0&2B4H}ux<@T`V
z6)V74qGe0ci#ha17jzf42gTWx#}(BJez&E9{I;bS(WI*<Ykmmn0cScqw!_s_nQncT
zzg+W(4EnkGg?LT^+yA4ep?Jy4#2)W}`mUxznu+&M+$-=cKB&HJi2~ji%&Q=w-698E
z&vkB&GOp%2|1CGBJija6>}Rhg9oKRf^{U_I9=|K!>>aLJ-fJ;Lu>+Lybp!mn>P-r9
z)m&T)57vL)`~O;v->uv%w^wxIT1-*Y1hs&_As@fnxmmTYiu1L2qr7>wk>5-8?-p(*
z)T`=tEj|SQqt^YdelxLLW!-CGA-bXd?;QNQ_RZsabx5y89~IN<Li?Xo^t*wZ(E4u!
c_ff0)A7@@e1rrq}000kl1)^%m=&wEh3uyeG+yDRo

literal 0
HcmV?d00001

diff --git a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties
index 3d6473dbf..28254fad5 100644
--- a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties
+++ b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties
@@ -3,11 +3,15 @@ Directive_parse_Fixed_Length=testData/Wrangler/parse_fixedlength_wrangle-cdap-da
 Directive_parse_hl7=testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json
 Directive_parse_Timestamp=testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json
 Directive_parse_Datetime=testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json
+Directive_parse_json=testData/Wrangler/parse_json_Wrangle-cdap-data-pipeline (1).json
+Directive_parse_xml=testData/Wrangler/parse_xmltojson_wrangle-cdap-data-pipeline.json
+Directive_parse_excel=testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json
 Directive_parse_csv=testData/Wrangler\
   /parse_csv_wrangle-cdap-data-pipeline.json
 bqSourceTable=dummy
 sourcePath=example/hello.csv
 gcsSourceBucket=dummy
+testFile=BQtesdata/BigQuery/test1.xlsx
 #bq queries file path
 
 CreateBQDataQueryFileFxdLen=BQtesdata/BigQuery/BigQueryCreateTableQueryFxdlen.txt
@@ -18,6 +22,10 @@ CreateBQDataQueryFileTimestamp=BQtesdata/BigQuery/BigQueryCreateTableQueryTimest
 InsertBQDataQueryFileTimestamp=BQtesdata/BigQuery/BigQueryInsertDataQueryTimestamp.txt
 CreateBQDataQueryFileDatetime=BQtesdata/BigQuery/BigQueryCreateTableQueryDatetime.txt
 InsertBQDataQueryFileDatetime=BQtesdata/BigQuery/BigQueryInsertDataQueryDatetime.txt
+CreateBQTableQueryFileJson=BQtesdata/BigQuery/BigQuerycreateTableQueryjson.txt
+InsertBQDataQueryFileJson=BQtesdata/BigQuery/BigQueryInsertDataQueryparsejson.txt
+CreateBQDataQueryFileXml=BQtesdata/BigQuery/BigQueryCreateTableQueryXml.txt
+InsertBQDataQueryFileXml=BQtesdata/BigQuery/BigQueryInsertDataQueryXml.txt
 CreateBQTableQueryFileCsv=BQtesdata/BigQuery/BigQueryCreateTableQueryCsv.txt
 InsertBQDataQueryFileCsv=BQtesdata/BigQuery/BigQueryInsertDataQueryCsv.txt
 
@@ -31,4 +39,7 @@ ExpectedDirective_parse_FixedLength=BQValidationExpectedFiles/Directive_parse_fi
 ExpectedDirective_parse_hl7=BQValidationExpectedFiles/Directive_parse_hl7
 ExpectedDirective_parse_Datetime=BQValidationExpectedFiles/Directive_parse_DateTime
 ExpectedDirective_parse_Timestamp=BQValidationExpectedFiles/Directive_parse_Timestamp
+ExpectedDirective_parse_json=BQValidationExpectedFiles/Directive_parse_json
+ExpectedDirective_parse_xml=BQValidationExpectedFiles/Directive_parse_xmltojson
+ExpectedDirective_parse_excel=BQValidationExpectedFiles/Directive_parse_excel
 ExpectedDirective_parse_csv=BQValidationExpectedFiles/Directive_parse_csv
diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json
new file mode 100644
index 000000000..614f18fdf
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json
@@ -0,0 +1,186 @@
+{
+    "name": "parse_excel_wrangler_copy",
+    "description": "Data Pipeline Application",
+    "artifact": {
+        "name": "cdap-data-pipeline",
+        "version": "6.10.0-SNAPSHOT",
+        "scope": "SYSTEM"
+    },
+    "config": {
+        "resources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "driverResources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "connections": [
+            {
+                "from": "GCSFile",
+                "to": "Wrangler"
+            },
+            {
+                "from": "Wrangler",
+                "to": "BigQuery"
+            }
+        ],
+        "postActions": [],
+        "properties": {},
+        "processTimingEnabled": true,
+        "stageLoggingEnabled": true,
+        "stages": [
+            {
+                "name": "GCSFile",
+                "plugin": {
+                    "name": "GCSFile",
+                    "type": "batchsource",
+                    "label": "GCSFile",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.23.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "format": "blob",
+                        "path": "gs://00000000-e2e-0014a44f-81be-4501-8360-0ddca1c39789/test1.xlsx",
+                        "fileEncoding": "UTF-8",
+                        "useConnection": "false",
+                        "referenceName": "test",
+                        "schema": "{\"type\":\"record\",\"name\":\"blob\",\"fields\":[{\"name\":\"body\",\"type\":\"bytes\"}]}",
+                        "project": "auto-detect",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "sampleSize": "1000",
+                        "filenameOnly": "false",
+                        "recursive": "false",
+                        "ignoreNonExistingFolders": "false",
+                        "encrypted": "false"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"blob\",\"fields\":[{\"name\":\"body\",\"type\":\"bytes\"}]}"
+                    }
+                ],
+                "id": "GCSFile",
+                "type": "batchsource",
+                "label": "GCSFile",
+                "icon": "fa-plug",
+                "$$hashKey": "object:475",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "496px",
+                    "top": "343px"
+                }
+            },
+            {
+                "name": "Wrangler",
+                "plugin": {
+                    "name": "Wrangler",
+                    "type": "transform",
+                    "label": "Wrangler",
+                    "artifact": {
+                        "name": "wrangler-transform",
+                        "version": "4.10.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "directives": "parse-as-excel :body '0' true\ncopy name copiedname\nmerge name bkd uniquenum ','\nrename bkd rollno\ndrop fwd\nswap id rollno\nsplit-to-rows :name 'o'\nfilter-rows-on condition-false rollno !~ '2.0'",
+                        "field": "*",
+                        "precondition": "false",
+                        "workspaceId": "667f9e85-6c36-4d38-ad48-ef85db7a04a2",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"int\",\"null\"]},{\"name\":\"rollno\",\"type\":[\"string\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"phone\",\"type\":[\"string\",\"null\"]},{\"name\":\"copiedname\",\"type\":[\"string\",\"null\"]},{\"name\":\"uniquenum\",\"type\":[\"string\",\"null\"]}]}",
+                        "on-error": "fail-pipeline"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"int\",\"null\"]},{\"name\":\"rollno\",\"type\":[\"string\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"phone\",\"type\":[\"string\",\"null\"]},{\"name\":\"copiedname\",\"type\":[\"string\",\"null\"]},{\"name\":\"uniquenum\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                ],
+                "inputSchema": [
+                    {
+                        "name": "GCSFile",
+                        "schema": "{\"type\":\"record\",\"name\":\"blob\",\"fields\":[{\"name\":\"body\",\"type\":\"bytes\"}]}"
+                    }
+                ],
+                "id": "Wrangler",
+                "type": "transform",
+                "label": "Wrangler",
+                "icon": "icon-DataPreparation",
+                "$$hashKey": "object:476",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "796px",
+                    "top": "343px"
+                }
+            },
+            {
+                "name": "BigQuery",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsink",
+                    "label": "BigQuery",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.23.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "project": "auto-detect",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "dataset": "Wrangler",
+                        "table": "extab34",
+                        "operation": "insert",
+                        "truncateTable": "false",
+                        "allowSchemaRelaxation": "false",
+                        "location": "US",
+                        "createPartitionedTable": "false",
+                        "partitioningType": "TIME",
+                        "partitionFilterRequired": "false",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"int\",\"null\"]},{\"name\":\"rollno\",\"type\":[\"string\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"phone\",\"type\":[\"string\",\"null\"]},{\"name\":\"copiedname\",\"type\":[\"string\",\"null\"]},{\"name\":\"uniquenum\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"int\",\"null\"]},{\"name\":\"rollno\",\"type\":[\"string\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"phone\",\"type\":[\"string\",\"null\"]},{\"name\":\"copiedname\",\"type\":[\"string\",\"null\"]},{\"name\":\"uniquenum\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                ],
+                "inputSchema": [
+                    {
+                        "name": "Wrangler",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"int\",\"null\"]},{\"name\":\"rollno\",\"type\":[\"string\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"phone\",\"type\":[\"string\",\"null\"]},{\"name\":\"copiedname\",\"type\":[\"string\",\"null\"]},{\"name\":\"uniquenum\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                ],
+                "id": "BigQuery",
+                "type": "batchsink",
+                "label": "BigQuery",
+                "icon": "fa-plug",
+                "$$hashKey": "object:477",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "1096px",
+                    "top": "343px"
+                }
+            }
+        ],
+        "schedule": "0 1 */1 * *",
+        "engine": "spark",
+        "numOfRecordsPreview": 100,
+        "rangeRecordsPreview": {
+            "min": 1,
+            "max": "5000"
+        },
+        "description": "Data Pipeline Application",
+        "maxConcurrentRuns": 1,
+        "pushdownEnabled": false,
+        "transformationPushdown": {}
+    },
+    "version": "2dd12daa-5395-11ee-9dac-000000d0cf32"
+}
\ No newline at end of file
diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_json_Wrangle-cdap-data-pipeline (1).json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_json_Wrangle-cdap-data-pipeline (1).json
new file mode 100644
index 000000000..ccc37ccef
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_json_Wrangle-cdap-data-pipeline (1).json	
@@ -0,0 +1,180 @@
+{
+    "name": "parse_json_Wrangle",
+    "description": "Data Pipeline Application",
+    "artifact": {
+        "name": "cdap-data-pipeline",
+        "version": "6.10.0-SNAPSHOT",
+        "scope": "SYSTEM"
+    },
+    "config": {
+        "resources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "driverResources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "connections": [
+            {
+                "from": "BigQueryTable",
+                "to": "Wrangler"
+            },
+            {
+                "from": "Wrangler",
+                "to": "BigQuery2"
+            }
+        ],
+        "postActions": [],
+        "properties": {},
+        "processTimingEnabled": true,
+        "stageLoggingEnabled": true,
+        "stages": [
+            {
+                "name": "BigQueryTable",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsource",
+                    "label": "BigQueryTable",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.23.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "dataset": "Wrangler",
+                        "table": "jstab",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"Body\",\"type\":\"string\"},{\"name\":\"json\",\"type\":\"string\"}]}",
+                        "project": "auto-detect",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "enableQueryingViews": "false"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"Body\",\"type\":\"string\"},{\"name\":\"json\",\"type\":\"string\"}]}"
+                    }
+                ],
+                "id": "BigQueryTable",
+                "type": "batchsource",
+                "label": "BigQueryTable",
+                "icon": "fa-plug",
+                "$$hashKey": "object:518",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "496px",
+                    "top": "327px"
+                }
+            },
+            {
+                "name": "Wrangler",
+                "plugin": {
+                    "name": "Wrangler",
+                    "type": "transform",
+                    "label": "Wrangler",
+                    "artifact": {
+                        "name": "wrangler-transform",
+                        "version": "4.10.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "directives": "parse-as-json :json 1\nltrim :Body\nset-column :desc concat(json_pet, \", \", body)\ncopy :json_name :copied\nswap :json_id :json_age\nmerge :json_id :json_name :json_id_json_name ,\nmask-number :json_pet 'testing'\ndrop json_height\nwrite-as-json-map :json_age\nrename json_id id",
+                        "field": "*",
+                        "precondition": "false",
+                        "workspaceId": "d496b8e4-ca6f-4d38-9877-d76bb52c218e",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Body\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_age\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_name\",\"type\":[{\"type\":\"record\",\"name\":\"json_name05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"json_pet\",\"type\":[\"string\",\"null\"]},{\"name\":\"desc\",\"type\":[\"string\",\"null\"]},{\"name\":\"copied\",\"type\":[{\"type\":\"record\",\"name\":\"copied05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"json_id_json_name\",\"type\":[\"string\",\"null\"]}]}",
+                        "on-error": "fail-pipeline"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Body\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_age\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_name\",\"type\":[{\"type\":\"record\",\"name\":\"json_name05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"json_pet\",\"type\":[\"string\",\"null\"]},{\"name\":\"desc\",\"type\":[\"string\",\"null\"]},{\"name\":\"copied\",\"type\":[{\"type\":\"record\",\"name\":\"copied05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"json_id_json_name\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                ],
+                "inputSchema": [
+                    {
+                        "name": "BigQueryTable",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"Body\",\"type\":\"string\"},{\"name\":\"json\",\"type\":\"string\"}]}"
+                    }
+                ],
+                "id": "Wrangler",
+                "type": "transform",
+                "label": "Wrangler",
+                "icon": "icon-DataPreparation",
+                "$$hashKey": "object:519",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "796px",
+                    "top": "327px"
+                }
+            },
+            {
+                "name": "BigQuery2",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsink",
+                    "label": "BigQuery2",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.23.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "project": "auto-detect",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "dataset": "Wrangler",
+                        "table": "jstabsupd",
+                        "operation": "insert",
+                        "truncateTable": "false",
+                        "allowSchemaRelaxation": "false",
+                        "location": "US",
+                        "createPartitionedTable": "false",
+                        "partitioningType": "TIME",
+                        "partitionFilterRequired": "false",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Body\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_age\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_name\",\"type\":[{\"type\":\"record\",\"name\":\"json_name05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"json_pet\",\"type\":[\"string\",\"null\"]},{\"name\":\"desc\",\"type\":[\"string\",\"null\"]},{\"name\":\"copied\",\"type\":[{\"type\":\"record\",\"name\":\"copied05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"json_id_json_name\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Body\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_age\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_name\",\"type\":[{\"type\":\"record\",\"name\":\"json_name05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"json_pet\",\"type\":[\"string\",\"null\"]},{\"name\":\"desc\",\"type\":[\"string\",\"null\"]},{\"name\":\"copied\",\"type\":[{\"type\":\"record\",\"name\":\"copied05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"json_id_json_name\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                ],
+                "inputSchema": [
+                    {
+                        "name": "Wrangler",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Body\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_age\",\"type\":[\"string\",\"null\"]},{\"name\":\"json_name\",\"type\":[{\"type\":\"record\",\"name\":\"json_name05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"json_pet\",\"type\":[\"string\",\"null\"]},{\"name\":\"desc\",\"type\":[\"string\",\"null\"]},{\"name\":\"copied\",\"type\":[{\"type\":\"record\",\"name\":\"copied05F0DF247CD8481657781C26E1595028\",\"fields\":[{\"name\":\"first\",\"type\":[\"string\",\"null\"]},{\"name\":\"last\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"json_id_json_name\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                ],
+                "id": "BigQuery2",
+                "type": "batchsink",
+                "label": "BigQuery2",
+                "icon": "fa-plug",
+                "$$hashKey": "object:520",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "1096px",
+                    "top": "327px"
+                }
+            }
+        ],
+        "schedule": "0 1 */1 * *",
+        "engine": "spark",
+        "numOfRecordsPreview": 100,
+        "rangeRecordsPreview": {
+            "min": 1,
+            "max": "5000"
+        },
+        "description": "Data Pipeline Application",
+        "maxConcurrentRuns": 1,
+        "pushdownEnabled": false,
+        "transformationPushdown": {}
+    },
+    "version": "516e13c0-4c05-11ee-b70a-0000001ecd5c"
+}
\ No newline at end of file
diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_xmltojson_wrangle-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_xmltojson_wrangle-cdap-data-pipeline.json
new file mode 100644
index 000000000..044e982e8
--- /dev/null
+++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_xmltojson_wrangle-cdap-data-pipeline.json
@@ -0,0 +1,362 @@
+{
+    "name": "parse_xmltojson_wrangle",
+    "description": "Data Pipeline Application",
+    "artifact": {
+        "name": "cdap-data-pipeline",
+        "version": "6.10.0-SNAPSHOT",
+        "scope": "SYSTEM"
+    },
+    "config": {
+        "resources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "driverResources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "connections": [
+            {
+                "from": "BigQueryTable",
+                "to": "Wrangler"
+            },
+            {
+                "from": "Wrangler",
+                "to": "BigQuery2"
+            }
+        ],
+        "postActions": [],
+        "properties": {},
+        "processTimingEnabled": true,
+        "stageLoggingEnabled": true,
+        "stages": [
+            {
+                "name": "BigQueryTable",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsource",
+                    "label": "BigQueryTable",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.23.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "dataset": "Wrangler",
+                        "table": "xmltabs",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"Email\",\"type\":\"string\"},{\"name\":\"xmldata\",\"type\":\"string\"}]}",
+                        "project": "auto-detect",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "enableQueryingViews": "false"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"Email\",\"type\":\"string\"},{\"name\":\"xmldata\",\"type\":\"string\"}]}"
+                    }
+                ],
+                "id": "BigQueryTable",
+                "type": "batchsource",
+                "label": "BigQueryTable",
+                "icon": "fa-plug",
+                "$$hashKey": "object:615",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "496px",
+                    "top": "343px"
+                },
+                "_backendProperties": {
+                    "schema": {
+                        "name": "schema",
+                        "description": "The schema of the table to read.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "viewMaterializationDataset": {
+                        "name": "viewMaterializationDataset",
+                        "description": "The dataset in the specified project where the temporary table should be created. Defaults to the same dataset in which the table is located.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "enableQueryingViews": {
+                        "name": "enableQueryingViews",
+                        "description": "Whether to allow querying views. Since BigQuery views are not materialized by default, querying them may have a performance overhead.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "serviceAccountJSON": {
+                        "name": "serviceAccountJSON",
+                        "description": "Content of the service account file.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "partitionTo": {
+                        "name": "partitionTo",
+                        "description": "It's inclusive partition end date. It should be a String with format \"yyyy-MM-dd\". This value is ignored if the table does not support partitioning.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "cmekKey": {
+                        "name": "cmekKey",
+                        "description": "The GCP customer managed encryption key (CMEK) name used to encrypt data written to any bucket, dataset or table created by the plugin. If the bucket, dataset or table already exists, this is ignored. More information can be found at https://cloud.google.com/data-fusion/docs/how-to/customer-managed-encryption-keys",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "useConnection": {
+                        "name": "useConnection",
+                        "description": "Whether to use an existing connection.",
+                        "type": "boolean",
+                        "required": false,
+                        "macroSupported": false,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "project": {
+                        "name": "project",
+                        "description": "Google Cloud Project ID. It can be found on the Dashboard in the Google Cloud Platform Console.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "viewMaterializationProject": {
+                        "name": "viewMaterializationProject",
+                        "description": "The project name where the temporary table should be created. Defaults to the same project in which the table is located.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "filter": {
+                        "name": "filter",
+                        "description": "The WHERE clause filters out rows by evaluating each row against boolean expression, and discards all rows that do not return TRUE (that is, rows that return FALSE or NULL).",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "bucket": {
+                        "name": "bucket",
+                        "description": "The Google Cloud Storage bucket to store temporary data in. Cloud Storage data will be deleted after it is loaded into BigQuery. If it is not provided, a unique bucket will be automatically created and then deleted after the run finishes. The service account must have permission to create buckets in the configured project.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "partitionFrom": {
+                        "name": "partitionFrom",
+                        "description": "It's inclusive partition start date. It should be a String with format \"yyyy-MM-dd\". This value is ignored if the table does not support partitioning.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "serviceFilePath": {
+                        "name": "serviceFilePath",
+                        "description": "Path on the local file system of the service account key used for authorization. Can be set to 'auto-detect' when running on a Dataproc cluster. When running on other clusters, the file must be present on every node in the cluster.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "serviceAccountType": {
+                        "name": "serviceAccountType",
+                        "description": "Service account type, file path where the service account is located or the JSON content of the service account.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "connection": {
+                        "name": "connection",
+                        "description": "The existing connection to use.",
+                        "type": "bigqueryconnectorconfig",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": [
+                            "serviceAccountJSON",
+                            "serviceFilePath",
+                            "project",
+                            "serviceAccountType",
+                            "datasetProject"
+                        ]
+                    },
+                    "datasetProject": {
+                        "name": "datasetProject",
+                        "description": "The project the dataset belongs to. This is only required if the dataset is not in the same project that the BigQuery job will run in. If no value is given, it will default to the configured project ID.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "dataset": {
+                        "name": "dataset",
+                        "description": "The dataset to write to. A dataset is contained within a specific project. Datasets are top-level containers that are used to organize and control access to tables and views.",
+                        "type": "string",
+                        "required": true,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "table": {
+                        "name": "table",
+                        "description": "The table to read from. A table contains individual records organized in rows. Each record is composed of columns (also called fields). Every table is defined by a schema that describes the column names, data types, and other information.",
+                        "type": "string",
+                        "required": true,
+                        "macroSupported": true,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    },
+                    "referenceName": {
+                        "name": "referenceName",
+                        "description": "This will be used to uniquely identify this source for lineage, annotating metadata, etc.",
+                        "type": "string",
+                        "required": false,
+                        "macroSupported": false,
+                        "macroEscapingEnabled": false,
+                        "children": []
+                    }
+                },
+                "description": "This source reads the entire contents of a BigQuery table. BigQuery is Google's serverless, highly scalable, enterprise data warehouse.Data is first written to a temporary location on Google Cloud Storage, then read into the pipeline from there.",
+                "selected": false
+            },
+            {
+                "name": "Wrangler",
+                "plugin": {
+                    "name": "Wrangler",
+                    "type": "transform",
+                    "label": "Wrangler",
+                    "artifact": {
+                        "name": "wrangler-transform",
+                        "version": "4.10.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "directives": "parse-xml-to-json :xmldata 1\nsplit-email :email\ntext-distance block email email_account distance\ntext-metric longest-common-subsequence email email_account distance2\nwrite-as-json-object :email_domain distance,email_account\nstemming :email\nsplit-to-rows :email_account '0'\nrename :email_account id",
+                        "field": "*",
+                        "precondition": "false",
+                        "workspaceId": "f076fbe0-b7f8-47bc-8cc1-2a40e04177e1",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Email\",\"type\":[\"string\",\"null\"]},{\"name\":\"xmldata_note\",\"type\":[{\"type\":\"record\",\"name\":\"xmldata_note69A9BFB19CE40D9BB21E66FF1DCB2823\",\"fields\":[{\"name\":\"heading\",\"type\":[\"string\",\"null\"]},{\"name\":\"from\",\"type\":[\"string\",\"null\"]},{\"name\":\"to\",\"type\":[\"string\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"email_domain\",\"type\":[{\"type\":\"record\",\"name\":\"email_domain53E9571E3B0C6D8ACD29805625EDE284\",\"fields\":[{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_account\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"distance2\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_porter\",\"type\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}]}",
+                        "on-error": "fail-pipeline"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Email\",\"type\":[\"string\",\"null\"]},{\"name\":\"xmldata_note\",\"type\":[{\"type\":\"record\",\"name\":\"xmldata_note69A9BFB19CE40D9BB21E66FF1DCB2823\",\"fields\":[{\"name\":\"heading\",\"type\":[\"string\",\"null\"]},{\"name\":\"from\",\"type\":[\"string\",\"null\"]},{\"name\":\"to\",\"type\":[\"string\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"email_domain\",\"type\":[{\"type\":\"record\",\"name\":\"email_domain53E9571E3B0C6D8ACD29805625EDE284\",\"fields\":[{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_account\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"distance2\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_porter\",\"type\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}]}"
+                    }
+                ],
+                "inputSchema": [
+                    {
+                        "name": "BigQueryTable",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"Email\",\"type\":\"string\"},{\"name\":\"xmldata\",\"type\":\"string\"}]}"
+                    }
+                ],
+                "id": "Wrangler",
+                "type": "transform",
+                "label": "Wrangler",
+                "icon": "icon-DataPreparation",
+                "$$hashKey": "object:616",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "796px",
+                    "top": "343px"
+                },
+                "selected": false
+            },
+            {
+                "name": "BigQuery2",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsink",
+                    "label": "BigQuery2",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.23.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "project": "auto-detect",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "dataset": "Wrangler",
+                        "table": "xmltabupd",
+                        "operation": "insert",
+                        "truncateTable": "false",
+                        "allowSchemaRelaxation": "false",
+                        "location": "US",
+                        "createPartitionedTable": "false",
+                        "partitioningType": "TIME",
+                        "partitionFilterRequired": "false",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Email\",\"type\":[\"string\",\"null\"]},{\"name\":\"xmldata_note\",\"type\":[{\"type\":\"record\",\"name\":\"xmldata_note69A9BFB19CE40D9BB21E66FF1DCB2823\",\"fields\":[{\"name\":\"heading\",\"type\":[\"string\",\"null\"]},{\"name\":\"from\",\"type\":[\"string\",\"null\"]},{\"name\":\"to\",\"type\":[\"string\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"email_domain\",\"type\":[{\"type\":\"record\",\"name\":\"email_domain53E9571E3B0C6D8ACD29805625EDE284\",\"fields\":[{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_account\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"distance2\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_porter\",\"type\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}]}"
+                    }
+                },
+                "outputSchema": [
+                    {
+                        "name": "etlSchemaBody",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Email\",\"type\":[\"string\",\"null\"]},{\"name\":\"xmldata_note\",\"type\":[{\"type\":\"record\",\"name\":\"xmldata_note69A9BFB19CE40D9BB21E66FF1DCB2823\",\"fields\":[{\"name\":\"heading\",\"type\":[\"string\",\"null\"]},{\"name\":\"from\",\"type\":[\"string\",\"null\"]},{\"name\":\"to\",\"type\":[\"string\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"email_domain\",\"type\":[{\"type\":\"record\",\"name\":\"email_domain53E9571E3B0C6D8ACD29805625EDE284\",\"fields\":[{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_account\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"distance2\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_porter\",\"type\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}]}"
+                    }
+                ],
+                "inputSchema": [
+                    {
+                        "name": "Wrangler",
+                        "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"Email\",\"type\":[\"string\",\"null\"]},{\"name\":\"xmldata_note\",\"type\":[{\"type\":\"record\",\"name\":\"xmldata_note69A9BFB19CE40D9BB21E66FF1DCB2823\",\"fields\":[{\"name\":\"heading\",\"type\":[\"string\",\"null\"]},{\"name\":\"from\",\"type\":[\"string\",\"null\"]},{\"name\":\"to\",\"type\":[\"string\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"email_domain\",\"type\":[{\"type\":\"record\",\"name\":\"email_domain53E9571E3B0C6D8ACD29805625EDE284\",\"fields\":[{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_account\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"distance\",\"type\":[\"float\",\"null\"]},{\"name\":\"distance2\",\"type\":[\"float\",\"null\"]},{\"name\":\"email_porter\",\"type\":[{\"type\":\"array\",\"items\":[\"string\",\"null\"]},\"null\"]}]}"
+                    }
+                ],
+                "id": "BigQuery2",
+                "type": "batchsink",
+                "label": "BigQuery2",
+                "icon": "fa-plug",
+                "$$hashKey": "object:617",
+                "isPluginAvailable": true,
+                "_uiPosition": {
+                    "left": "1096px",
+                    "top": "343px"
+                },
+                "selected": false
+            }
+        ],
+        "schedule": "0 1 */1 * *",
+        "engine": "spark",
+        "numOfRecordsPreview": 100,
+        "rangeRecordsPreview": {
+            "min": 1,
+            "max": "5000"
+        },
+        "maxConcurrentRuns": 1,
+        "pushdownEnabled": false,
+        "transformationPushdown": {}
+    },
+    "version": "fe20077f-4e31-11ee-b01d-000000d34db8"
+}
\ No newline at end of file