diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryBase.java b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryBase.java index 23a948f2f..d4ae865c8 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryBase.java +++ b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryBase.java @@ -94,14 +94,14 @@ public void getCountOfNoOfRecordsTransferredToTargetBigQueryTable() throws IOExc } @Then("Validate records transferred to target table is equal to number of records from source table " + - "with filter {string}") + "with filter {string}") public void validateRecordsTransferredToTargetTableIsEqualToNumberOfRecordsFromSourceTableWithFilter(String filter) - throws IOException, InterruptedException { + throws IOException, InterruptedException { String projectId = (PluginPropertyUtils.pluginProp("projectId")); String datasetName = (PluginPropertyUtils.pluginProp("dataset")); int countRecordsTarget = BigQueryClient.countBqQuery(TestSetupHooks.bqTargetTable); String selectQuery = "SELECT count(*) FROM `" + projectId + "." + datasetName + "." + - TestSetupHooks.bqTargetTable + "` WHERE " + PluginPropertyUtils.pluginProp(filter); + TestSetupHooks.bqTargetTable + "` WHERE " + PluginPropertyUtils.pluginProp(filter); Optional result = BigQueryClient.getSoleQueryResult(selectQuery); int count = result.map(Integer::parseInt).orElse(0); BeforeActions.scenario.write("Number of records transferred with respect to filter:" + count); @@ -110,7 +110,7 @@ public void validateRecordsTransferredToTargetTableIsEqualToNumberOfRecordsFromS @Then("Validate partition date in output partitioned table") public void validatePartitionDateInOutputPartitionedTable() - throws IOException, InterruptedException { + throws IOException, InterruptedException { Optional result = BigQueryClient .getSoleQueryResult("SELECT distinct _PARTITIONDATE as pt FROM `" + (PluginPropertyUtils.pluginProp("projectId")) + "." + @@ -233,10 +233,10 @@ public void verifyTheBigQueryValidationErrorMessageForInvalidProperty(String pro .errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_CHUNKSIZE); } else if (property.equalsIgnoreCase("bucket")) { expectedErrorMessage = PluginPropertyUtils - .errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_TEMPORARY_BUCKET); + .errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_TEMPORARY_BUCKET); } else if (property.equalsIgnoreCase("table")) { expectedErrorMessage = PluginPropertyUtils - .errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_TABLE_NAME); + .errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_TABLE_NAME); } else { expectedErrorMessage = PluginPropertyUtils.errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_PROPERTY). replaceAll("PROPERTY", property.substring(0, 1).toUpperCase() + property.substring(1)); @@ -260,5 +260,4 @@ public void validateRecordsTransferredToTargetTableIsEqualToNumberOfRecordsFromS BeforeActions.scenario.write("Number of records transferred from source table to target table:" + count); Assert.assertEquals(count, countRecordsTarget); } - } diff --git a/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java b/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java index 10a848a9b..d54c4e176 100644 --- a/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java +++ b/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java @@ -39,8 +39,7 @@ public enum CdfPluginPropertyLocator { GCS_MOVE_DESTINATION_PATH("destPath"), PARTITION_START_DATE("partitionFrom"), PARTITION_END_DATE("partitionTo"), - FILTER("filter"), - OUTPUT_SCHEMA("Output Schema-macro-input"); + FILTER("filter"); public String pluginProperty; CdfPluginPropertyLocator(String property) { @@ -79,7 +78,6 @@ public enum CdfPluginPropertyLocator { .put("gcsMoveSourcePath", CdfPluginPropertyLocator.GCS_MOVE_SOURCE_PATH) .put("gcsMoveDestinationPath", CdfPluginPropertyLocator.GCS_MOVE_DESTINATION_PATH) .put("filter", CdfPluginPropertyLocator.FILTER) - .put("Output Schema-macro-input", CdfPluginPropertyLocator.OUTPUT_SCHEMA) .put("partitionFrom", CdfPluginPropertyLocator.PARTITION_START_DATE) .put("partitionTo", CdfPluginPropertyLocator.PARTITION_END_DATE) .build(); diff --git a/src/e2e-test/resources/errorMessage.properties b/src/e2e-test/resources/errorMessage.properties index 671bfb630..6c8108fac 100644 --- a/src/e2e-test/resources/errorMessage.properties +++ b/src/e2e-test/resources/errorMessage.properties @@ -33,7 +33,7 @@ errorMessageMultipleFileWithoutClearDefaultSchema=Found a row with 4 fields when errorMessageInvalidSourcePath=Invalid bucket name in path 'abc@'. Bucket name should errorMessageInvalidDestPath=Invalid bucket name in path 'abc@'. Bucket name should errorMessageInvalidEncryptionKey=CryptoKeyName.parse: formattedString not in valid format: Parameter "abc@" must be -errorMessageIncorrectPartitionStartDate=10-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd -errorMessageIncorrectPartitionEndDate=11-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd +errorMessageIncorrectPartitionStartDate=13-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd +errorMessageIncorrectPartitionEndDate=14-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd errorMessageIncorrectReferenceName=Invalid reference name 'invalidRef&^*&&*'. Supported characters are: letters, numbers, and '_', '-', '.', or '$'. errorLogsMessageInvalidFilter=Spark Program 'phase-1' failed. diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 475ec0fd7..ee9ebd7a4 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -370,17 +370,17 @@ bqTargetTable=dummy bqTargetTable2=dummy bqmtTargetTable=tabA bqmtTargetTable2=tabB -bqStartDate=2025-01-10 -bqEndDate=2025-01-11 -partitionFrom=2025-01-10 -partitionTo=2025-01-11 +bqStartDate=2025-01-13 +bqEndDate=2025-01-14 +partitionFrom=2025-01-113 +partitionTo=2025-01-14 filter=Id=20 bqInvalidReferenceName=invalidRef&^*&&* OutputSchema={ "type": "record", "name": "text", "fields": [{ "name": "Id", "type": "long" }, { "name": "Value", "type": "long" }, \ { "name": "UID", "type": "string" } ] } incorrectFilter=%%%% -bqIncorrectFormatStartDate=10-01-2025 -bqIncorrectFormatEndDate=11-01-2025 +bqIncorrectFormatStartDate=13-01-2025 +bqIncorrectFormatEndDate=14-01-2025 ## BQMT-PLUGIN-PROPERTIES-END ##CLOUDBIGTABLE-PLUGIN-PROPERTIES-START