debug

wrangler-transform/src/e2e-test/features/Wrangler/DataTypeParsers.feature

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-@Wrangler
+@Wranglers
 Feature:  datatype parsers
 
   @BQ_SOURCE_TS_TEST @BQ_SINK_TEST

wrangler-transform/src/e2e-test/features/Wrangler/ParseAsAvro.feature

@@ -15,7 +15,7 @@
 @Wrangler
 Feature:  Wrangler - Run time scenarios for Parse Avro
 
-  @BQ_SOURCE_AVRO_TEST @BQ_SINK_TEST
+  @BQ_SOURCE_AVRO_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST
   Scenario: To verify User is able to run a pipeline using parse avro directive
     Given Open Datafusion Project to configure pipeline
     Then Click on the Plus Green Button to import the pipelines

wrangler-transform/src/e2e-test/features/Wrangler/ParseAsCsv.feature

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-@Wrangler
+@Wranglers
 Feature:  Wrangler - Run time scenarios
 
   @BQ_SOURCE_CSV_TEST @BQ_SINK_TEST

wrangler-transform/src/e2e-test/features/Wrangler/ParseAsFixedLength.feature

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-@Wrangler
+@Wranglers
 Feature:  parse as fixed length
 
   @BQ_SOURCE_FXDLEN_TEST @BQ_SINK_TEST

wrangler-transform/src/e2e-test/features/Wrangler/ParseAsHl7.feature

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-@Wrangler
+@Wranglers
 Feature:  parse as HL7
 
   @BQ_SOURCE_HL7_TEST @BQ_SINK_TEST

wrangler-transform/src/e2e-test/features/Wrangler/ParseAsLog.feature

@@ -15,7 +15,7 @@
 @Wrangler
 Feature:  Wrangler - Run time scenarios for Parse Log
 
-  @BQ_SOURCE_LOG_TEST @BQ_SINK_TEST
+  @BQ_SOURCE_LOG_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST
   Scenario: To verify User is able to run a pipeline using parse log directive
     Given Open Datafusion Project to configure pipeline
     Then Click on the Plus Green Button to import the pipelines

wrangler-transform/src/e2e-test/java/io/cdap/plugin/wrangler/actions/ValidationHelper.java

@@ -61,36 +61,21 @@ public static void getFileData(String fileName, Map<String, JsonObject> fileMap)
       String line;
       while ((line = br.readLine()) != null) {
         JsonObject json = gson.fromJson(line, JsonObject.class);
-
-        // Check if "id" or "ID" (case-insensitive) exists in the JSON object
-        boolean idFound = false;
-        if (json.has("id")) {
+        if (json.has("id")) { // Check if the JSON object has the "id" key
           JsonElement idElement = json.get("id");
           if (idElement.isJsonPrimitive()) {
             String idKey = idElement.getAsString();
             fileMap.put(idKey, json);
-            idFound = true;
           } else {
-            LOG.error("Data Mismatched");
+            Log.error("ID key not found");
           }
         }
-
-        if (!idFound && json.has("ID")) {
-          JsonElement idElement = json.get("ID");
-          if (idElement.isJsonPrimitive()) {
-            String idKey = idElement.getAsString();
-            fileMap.put(idKey, json);
-          } else {
-            LOG.error("Data Mismatched");
-          }
-        } else if (!idFound) {
-          LOG.error("ID Key not found in JSON object");
-        }
       }
     } catch (IOException e) {
       System.err.println("Error reading the file: " + e.getMessage());
     }
   }
+
   private static void getBigQueryTableData(String targetTable, Map<String, JsonObject> bigQueryMap)
     throws IOException, InterruptedException {
     String dataset = PluginPropertyUtils.pluginProp("dataset");
@@ -100,27 +85,15 @@ private static void getBigQueryTableData(String targetTable, Map<String, JsonObj
 
     for (FieldValueList row : result.iterateAll()) {
       JsonObject json = gson.fromJson(row.get(0).getStringValue(), JsonObject.class);
-      boolean idFound = false;
-      if (json.has("id")) {
+      if (json.has("id")) { // Check if the JSON object has the "id" key
         JsonElement idElement = json.get("id");
-        if (idElement.isJsonPrimitive()) {
-          String idKey = idElement.getAsString();
-          bigQueryMap.put(idKey, json);
-          idFound = true;
-        } else {
-          LOG.error("Data Mismatched");
-        }
-      }
-
-      if (!idFound && json.has("ID")) {
-        JsonElement idElement = json.get("ID");
         if (idElement.isJsonPrimitive()) {
           String idKey = idElement.getAsString();
           bigQueryMap.put(idKey, json);
         } else {
           LOG.error("Data Mismatched");
         }
-      } else if (!idFound) {
+      } else {
         LOG.error("ID Key not found in JSON object");
       }
     }

wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_log

@@ -1 +1 @@
-{"Body":"127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326","Body_icd_9_description":"Body Post Catalog lookup","ID":1,"bytes_response_body_bytes":"2326","bytes_response_body_bytes_last":"2326","http_firstline_request_firstline":"GET /apache_pb.gif HTTP/1.0","http_method_request_firstline_method":"GET","http_path_request_firstline_uri_path":"/apache_pb.gif","http_protocol_request_firstline_protocol":"HTTP","http_protocol_version_request_firstline_protocol":"HTTP/1.0","http_protocol_version_request_firstline_protocol_version":"1.0","http_ref_request_firstline_uri_ref":"no value","http_uri_request_firstline_uri":"/apache_pb.gif","ip_connection_client_host":"127.0.0.1","ip_connection_client_host_1":"127.0.0.1","ip_connection_client_host_last":"127.0.0.1","string_connection_client_user":"frank","string_connection_client_user_last":"frank","string_request_status_last":"200","time_date_request_receive_time_date":"2000-10-10","time_date_request_receive_time_date_utc":"2000-10-10","time_date_request_receive_time_last_date":"2000-10-10","time_date_request_receive_time_last_date_utc":"2000-10-10","time_day_request_receive_time_day":"10","time_day_request_receive_time_day_utc":"10","time_day_request_receive_time_last_day":10,"time_day_request_receive_time_last_day_utc":"10","time_epoch_request_receive_time_epoch":"971211336000","time_epoch_request_receive_time_last_epoch":"971211336000","time_hour_request_receive_time_hour":"13","time_hour_request_receive_time_hour_utc":"20","time_hour_request_receive_time_last_hour":"13","time_hour_request_receive_time_last_hour_utc":"20","time_millisecond_request_receive_time_last_millisecond":"0","time_millisecond_request_receive_time_last_millisecond_utc":"0","time_millisecond_request_receive_time_millisecond":"0","time_millisecond_request_receive_time_millisecond_utc":"0","time_minute_request_receive_time_last_minute":"55","time_minute_request_receive_time_last_minute_utc":"55","time_minute_request_receive_time_minute":"55","time_minute_request_receive_time_minute_utc":"55","time_month_request_receive_time_last_month":"10","time_month_request_receive_time_last_month_utc":"10","time_month_request_receive_time_month":"10","time_month_request_receive_time_month_utc":"10","time_monthname_request_receive_time_last_monthname":"October","time_monthname_request_receive_time_last_monthname_utc":"October","time_monthname_request_receive_time_monthname":"October","time_monthname_request_receive_time_monthname_utc":"October","time_second_request_receive_time_last_second":"36","time_second_request_receive_time_last_second_utc":"36","time_second_request_receive_time_second":"36","time_second_request_receive_time_second_utc":"36","time_stamp_request_receive_time":"10/Oct/2000:13:55:36 -0700","time_stamp_request_receive_time_last":"10/Oct/2000:13:55:36 -0700","time_time_request_receive_time_last_time":"13:55:36","time_time_request_receive_time_last_time_utc":"20:55:36","time_time_request_receive_time_time":"13:55:36","time_time_request_receive_time_time_utc":"20:55:36","time_week_request_receive_time_last_weekofweekyear":"41","time_week_request_receive_time_last_weekofweekyear_utc":"41","time_week_request_receive_time_weekofweekyear":"41","time_week_request_receive_time_weekofweekyear_utc":"41","time_year_request_receive_time_last_weekyear":"2000","time_year_request_receive_time_last_weekyear_utc":"2000","time_year_request_receive_time_last_year":"2000","time_year_request_receive_time_last_year_utc":"2000","time_year_request_receive_time_weekyear":"2000","time_year_request_receive_time_weekyear_utc":"2000","time_year_request_receive_time_year":"2000","time_year_request_receive_time_year_utc":"2000"}
+{"Body_icd_9_description":"Body Post Catalog lookup","body":"127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326","bytes_response_body_bytes":"2326","bytes_response_body_bytes_last":"2326","http_firstline_request_firstline":"GET /apache_pb.gif HTTP/1.0","http_method_request_firstline_method":"GET","http_path_request_firstline_uri_path":"/apache_pb.gif","http_protocol_request_firstline_protocol":"HTTP","http_protocol_version_request_firstline_protocol":"HTTP/1.0","http_protocol_version_request_firstline_protocol_version":"1.0","http_ref_request_firstline_uri_ref":"no value","http_uri_request_firstline_uri":"/apache_pb.gif","id":1,"ip_connection_client_host":"127.0.0.1","ip_connection_client_host_1":"127.0.0.1","ip_connection_client_host_last":"127.0.0.1","string_connection_client_user":"frank","string_connection_client_user_last":"frank","string_request_status_last":"200","time_date_request_receive_time_date":"2000-10-10","time_date_request_receive_time_date_utc":"2000-10-10","time_date_request_receive_time_last_date":"2000-10-10","time_date_request_receive_time_last_date_utc":"2000-10-10","time_day_request_receive_time_day":"10","time_day_request_receive_time_day_utc":"10","time_day_request_receive_time_last_day":10,"time_day_request_receive_time_last_day_utc":"10","time_epoch_request_receive_time_epoch":"971211336000","time_epoch_request_receive_time_last_epoch":"971211336000","time_hour_request_receive_time_hour":"13","time_hour_request_receive_time_hour_utc":"20","time_hour_request_receive_time_last_hour":"13","time_hour_request_receive_time_last_hour_utc":"20","time_millisecond_request_receive_time_last_millisecond":"0","time_millisecond_request_receive_time_last_millisecond_utc":"0","time_millisecond_request_receive_time_millisecond":"0","time_millisecond_request_receive_time_millisecond_utc":"0","time_minute_request_receive_time_last_minute":"55","time_minute_request_receive_time_last_minute_utc":"55","time_minute_request_receive_time_minute":"55","time_minute_request_receive_time_minute_utc":"55","time_month_request_receive_time_last_month":"10","time_month_request_receive_time_last_month_utc":"10","time_month_request_receive_time_month":"10","time_month_request_receive_time_month_utc":"10","time_monthname_request_receive_time_last_monthname":"October","time_monthname_request_receive_time_last_monthname_utc":"October","time_monthname_request_receive_time_monthname":"October","time_monthname_request_receive_time_monthname_utc":"October","time_second_request_receive_time_last_second":"36","time_second_request_receive_time_last_second_utc":"36","time_second_request_receive_time_second":"36","time_second_request_receive_time_second_utc":"36","time_stamp_request_receive_time":"10/Oct/2000:13:55:36 -0700","time_stamp_request_receive_time_last":"10/Oct/2000:13:55:36 -0700","time_time_request_receive_time_last_time":"13:55:36","time_time_request_receive_time_last_time_utc":"20:55:36","time_time_request_receive_time_time":"13:55:36","time_time_request_receive_time_time_utc":"20:55:36","time_week_request_receive_time_last_weekofweekyear":"41","time_week_request_receive_time_last_weekofweekyear_utc":"41","time_week_request_receive_time_weekofweekyear":"41","time_week_request_receive_time_weekofweekyear_utc":"41","time_year_request_receive_time_last_weekyear":"2000","time_year_request_receive_time_last_weekyear_utc":"2000","time_year_request_receive_time_last_year":"2000","time_year_request_receive_time_last_year_utc":"2000","time_year_request_receive_time_weekyear":"2000","time_year_request_receive_time_weekyear_utc":"2000","time_year_request_receive_time_year":"2000","time_year_request_receive_time_year_utc":"2000"}

wrangler-transform/src/e2e-test/resources/pluginParameters.properties

@@ -6,7 +6,7 @@ Directive_parse_Datetime=testData/Wrangler/parse_datetime_wrangle-cdap-data-pipe
 Directive_parse_csv=testData/Wrangler\
   /parse_csv_wrangle-cdap-data-pipeline.json
 Directive_parse_avro=testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json
-Directive_parse_log=testData/Wrangler/logParser-cdap-data-pipeline (1).json
+Directive_parse_log=testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json
 bqSourceTable=dummy
 sourcePath=example/hello.csv
 gcsSourceBucket=dummy