From b98cbb3d54cb58ceb7b181dc2e624e5331d7a325 Mon Sep 17 00:00:00 2001 From: bharatgulati Date: Wed, 21 Jun 2023 18:03:57 +0530 Subject: [PATCH] Debug - Record missmatch with complete suite. --- .../e2e-test/features/mssql/Datatype.feature | 5 +- .../features/mssql/DatatypeDateTime.feature | 5 +- .../features/mssql/DatatypeImage.feature | 5 +- .../mssql/DatatypeUniqueIdentifier.feature | 5 +- .../src/e2e-test/features/mssql/Mssql.feature | 5 +- .../mssql/mssql sink/DesignTime.feature | 8 +- .../mssql sink/DesignTimeValidation.feature | 12 +- .../features/mssql/mssql sink/RunTime.feature | 4 +- .../mssql sink/RunTimeWithMacros.feature | 10 +- .../mssql/mssql source/DesignTime.feature | 8 +- .../mssql source/DesignTimeValidation.feature | 20 +- .../mssql source/DesignTimeWithMacros.feature | 8 +- .../mssql/mssql source/RunTime.feature | 10 +- .../mssql/mssql source/RunTimeWMacros.feature | 111 +++++++ .../mssql source/RunTimeWithMacros.feature | 308 +++++++++--------- .../java/io.cdap.plugin/MssqlClient.java | 18 +- .../common.stepsdesign/TestSetupHooks.java | 192 ++++++++--- .../io.cdap.plugin/mssql/BQValidation.java | 14 +- .../mssql/stepsdesign/Mssql.java | 15 +- .../resources/errorMessage.properties | 4 +- .../resources/pluginParameters.properties | 2 + 21 files changed, 517 insertions(+), 252 deletions(-) create mode 100644 mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWMacros.feature diff --git a/mssql-plugin/src/e2e-test/features/mssql/Datatype.feature b/mssql-plugin/src/e2e-test/features/mssql/Datatype.feature index 97434c41e..a2f78e1ec 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/Datatype.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/Datatype.feature @@ -14,9 +14,10 @@ # the License. # -@Mssql +@Mssql @Mssql_Required Feature: Mssql - Verify Mssql source data transfer - @MSSQL_SOURCE_DATATYPES_TEST @MSSQL_SINK_TEST @Mssql_Required + + @MSSQL_SOURCE_DATATYPES_TEST @MSSQL_TARGET_DATATYPES_TEST Scenario: To verify data is getting transferred from Mssql to Mssql successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/DatatypeDateTime.feature b/mssql-plugin/src/e2e-test/features/mssql/DatatypeDateTime.feature index 0e82b034e..5f574aa7c 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/DatatypeDateTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/DatatypeDateTime.feature @@ -14,9 +14,10 @@ # the License. # -@Mssql +@Mssql @Mssql_Required Feature: Mssql - Verify Mssql source data transfer - @MSSQL_SOURCE_DATATYPES_DATETIME_TEST @MSSQL_SINK_TEST @Mssql_Required + + @MSSQL_SOURCE_DATATYPES_DATETIME_TEST @MSSQL_TARGET_DATATYPES_DATETIME_TEST Scenario: To verify data is getting transferred from Mssql to Mssql successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/DatatypeImage.feature b/mssql-plugin/src/e2e-test/features/mssql/DatatypeImage.feature index 7d2572e78..332e25216 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/DatatypeImage.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/DatatypeImage.feature @@ -14,9 +14,10 @@ # the License. # -@Mssql +@Mssql @Mssql_Required Feature: Mssql - Verify Mssql source data transfer - @MSSQL_SOURCE_DATATYPES_IMAGE_TEST @MSSQL_SINK_TEST @Mssql_Required + + @MSSQL_SOURCE_DATATYPES_IMAGE_TEST @MSSQL_TARGET_DATATYPES_IMAGE_TEST Scenario: To verify data is getting transferred from Mssql to Mssql successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/DatatypeUniqueIdentifier.feature b/mssql-plugin/src/e2e-test/features/mssql/DatatypeUniqueIdentifier.feature index 02a1c3f48..b0a915269 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/DatatypeUniqueIdentifier.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/DatatypeUniqueIdentifier.feature @@ -14,9 +14,10 @@ # the License. # -@Mssql +@Mssql @Mssql_Required Feature: Mssql - Verify Mssql source data transfer - @MSSQL_SOURCE_DATATYPES_UIDTYPE_TEST @MSSQL_SINK_TEST @Mssql_Required + + @MSSQL_SOURCE_DATATYPES_UIDTYPE_TEST @MSSQL_TARGET_DATATYPES_UIDTYPE_TEST Scenario: To verify data is getting transferred from Mssql to Mssql successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/Mssql.feature b/mssql-plugin/src/e2e-test/features/mssql/Mssql.feature index 0e9c0de0d..1449a1e95 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/Mssql.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/Mssql.feature @@ -14,9 +14,10 @@ # the License. # -@Mssql +@Mssql @Mssql_Required Feature: Mssql - Verify Mssql source data transfer - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required + + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: To verify data is getting transferred from Mssql to Mssql successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTime.feature index 27a4914de..34f03dc26 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTime.feature @@ -12,10 +12,10 @@ ## License for the specific language governing permissions and limitations under # the License.. -@Mssql +@Mssql @Mssql_Required Feature: MsSQL Server Sink - Design time scenarios - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST + @MSSQL_AS_TARGET Scenario: Verify user is able to validate the sink plugin with connection and basic details for connectivity Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Sink" @@ -33,7 +33,7 @@ Feature: MsSQL Server Sink - Design time scenarios Then Validate "SQL Server" plugin properties Then Close the Plugin Properties page - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST + @MSSQL_AS_TARGET Scenario: To verify Mssql source plugin validation with connection arguments Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Sink" @@ -52,7 +52,7 @@ Feature: MsSQL Server Sink - Design time scenarios Then Validate "SQL Server" plugin properties Then Close the Plugin Properties page - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST + @MSSQL_AS_TARGET Scenario: To verify Mssql source plugin validation with advanced details Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Sink" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature index 092f1cbd5..1385cc8c1 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature @@ -12,10 +12,10 @@ ## License for the specific language governing permissions and limitations under # the License.. -@Mssql +@Mssql @Mssql_Required Feature: Mssql source- Verify Mssql source plugin design time validation scenarios - @MSSQL_SOURCE_TEST @Mssql_Required @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: To verify Mssql sink plugin validation error message with invalid database Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -47,7 +47,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari And Click on the Validate button Then Verify that the Plugin is displaying an error message: "errorMessageInvalidSinkDatabase" on the header - @MSSQL_SOURCE_TEST @Mssql_Required @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: To Verify Table Name Field validation error message with invalid test data Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -79,7 +79,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari And Click on the Validate button Then Verify that the Plugin Property: "tableName" is displaying an in-line error message: "errorMessageInvalidSinkTableName" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: To verify Mssql sink plugin validation error message with invalid reference name Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -111,7 +111,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari And Click on the Validate button Then Verify that the Plugin Property: "referenceName" is displaying an in-line error message: "errorMessageMssqlInvalidReferenceName" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: Verify the Username field validation error message with blank value Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -142,7 +142,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari And Click on the Validate button Then Verify that the Plugin Property: "user" is displaying an in-line error message: "errorMessageBlankUsername" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: Verify the Host validation error when values are not provided Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature index 476b4046a..31e731858 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations under # the License. -@Mssql +@Mssql @Mssql_Required Feature: Mssql - Verify Mssql sink data transfer - @BQ_SOURCE_TEST @MSSQL_TEST_TABLE @Mssql_Required @Plugin-1526 + @BQ_SOURCE_TEST @MSSQL_TEST_TABLE @Plugin-1526 Scenario: Verify User is able to preview and deploy the pipeline when the data is transferred from BigQuery to Mssql Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature index addfd3087..775df040e 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations under # the License. -@Mssql +@Mssql @Mssql_Required Feature: Mssql Sink - Run time scenarios (macro) - @BQ_SOURCE_TEST @MSSQL_TEST_TABLE @Mssql_Required + @BQ_SOURCE_TEST @MSSQL_TEST_TABLE Scenario: Verify User is able to preview and deploy the pipeline when the data is transferred from BigQuery to Mssql using macros Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -66,7 +66,7 @@ Feature: Mssql Sink - Run time scenarios (macro) Then Verify the pipeline status is "Succeeded" Then Validate the values of records transferred to target MsSql table is equal to the values from source BigQuery table - @BQ_SOURCE_TEST @MSSQL_TEST_TABLE @Mssql_Required + @BQ_SOURCE_TEST @MSSQL_TEST_TABLE @Plugin-1526 Scenario: Verify User is able to preview and deploy the pipeline when the data is transferred from BigQuery to Mssql using basic section as macros Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -113,7 +113,7 @@ Feature: Mssql Sink - Run time scenarios (macro) Then Verify the pipeline status is "Succeeded" Then Validate the values of records transferred to target MsSql table is equal to the values from source BigQuery table - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: Verify pipeline failure message in logs when user provides invalid Table Name of plugin with Macros Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -156,7 +156,7 @@ Feature: Mssql Sink - Run time scenarios (macro) | Level | Message | | ERROR | errorMessageInvalidTableName | - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: Verify pipeline failure message in logs when user provides invalid Credentials for connection with Macros Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTime.feature index 8b6d8cb77..28b0ed248 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTime.feature @@ -12,10 +12,10 @@ ## License for the specific language governing permissions and limitations under # the License.. -@Mssql +@Mssql @Mssql_Required Feature: MsSQL Server Source - Design time scenarios - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: Verify user is able to validate the plugin with connection and basic details for connectivity Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -35,7 +35,7 @@ Feature: MsSQL Server Source - Design time scenarios Then Validate "SQL Server" plugin properties Then Close the Plugin Properties page - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation with connection arguments Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -56,7 +56,7 @@ Feature: MsSQL Server Source - Design time scenarios Then Validate "SQL Server" plugin properties Then Close the Plugin Properties page - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation with advanced details Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature index ff93d34c8..74277fa77 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature @@ -12,10 +12,10 @@ ## License for the specific language governing permissions and limitations under # the License.. -@Mssql +@Mssql @Mssql_Required Feature: Mssql source- Verify Mssql source plugin design time validation scenarios - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message with invalid database Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -33,7 +33,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin is displaying an error message: "errorMessageInvalidSourceDatabase" on the header - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message with invalid import query Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -52,7 +52,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin Property: "importQuery" is displaying an in-line error message: "errorMessageInvalidImportQuery" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message with invalid reference name Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -70,7 +70,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin Property: "referenceName" is displaying an in-line error message: "errorMessageMssqlInvalidReferenceName" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message with blank username Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -87,7 +87,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin Property: "user" is displaying an in-line error message: "errorMessageBlankUsername" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message with blank password Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -104,7 +104,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin is displaying an error message: "errorMessageBlankPassword" on the header - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message when fetch size is changed to zero Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -143,7 +143,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessageBlankSplitBy" Then Verify that the Plugin Property: "splitBy" is displaying an in-line error message: "errorMessageBlankSplitBy" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message when number of Split value is changed to zero Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -162,7 +162,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessageInvalidNumberOfSplits" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message when number of Split value is not a number Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -181,7 +181,7 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessageNumberOfSplitNotNumber" - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: To verify Mssql source plugin validation error message with blank bounding query Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeWithMacros.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeWithMacros.feature index aaebbeb8d..3534df927 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeWithMacros.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeWithMacros.feature @@ -12,10 +12,10 @@ ## License for the specific language governing permissions and limitations under # the License.. -@Mssql +@Mssql @Mssql_Required Feature: MSSQL Server - Design time scenarios (macro) - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: Verify user should be able to validate plugin with macros for Connection section Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -31,13 +31,13 @@ Feature: MSSQL Server - Design time scenarios (macro) Then Enter textarea plugin property: "importQuery" with value: "selectQuery" Then Validate "SQL Server" plugin properties - @MSSQL_SOURCE_TEST @Mssql_Required + @MSSQL_AS_SOURCE Scenario: Verify user should be able to validate plugin with macros for Basic section Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" When Select plugin: "SQL Server" from the plugins list as: "Source" Then Navigate to the properties page of plugin: "SQL Server" - Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "sqlserver42" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields Then Select radio button plugin property: "authenticationType" with value: "SQL Login" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature index 9ba85482d..ffc8c385f 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations under # the License. -@Mssql +@Mssql @Mssql_Required Feature: Mssql - Verify Mssql source data transfer - @MSSQL_SOURCE_DATATYPES_TEST @Mssql_Required @BQ_SINK @Plugin-1526 + @MSSQL_SOURCE_DATATYPES_TEST @BQ_AS_SINK @Plugin-1526 Scenario: Verify User is able to preview and deploy the pipeline when the data is transferred from Mssql to BigQuery with multiple datatypes Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -59,7 +59,7 @@ Feature: Mssql - Verify Mssql source data transfer Then Close the pipeline logs Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table - @MSSQL_SOURCE_TEST @BQ_SINK @Mssql_Required @Plugin-1526 + @MSSQL_AS_SOURCE @BQ_AS_SINK @Plugin-1526 Scenario: To verify data is getting transferred from Mssql to BigQuery successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -103,7 +103,7 @@ Feature: Mssql - Verify Mssql source data transfer Then Close the pipeline logs Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table - @MSSQL_SOURCE_TEST @BQ_SINK @Plugin-1526 + @MSSQL_AS_SOURCE @BQ_AS_SINK @Plugin-1526 Scenario: Verify User is able to preview and deploy the pipeline when connection arguments are set Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -148,7 +148,7 @@ Feature: Mssql - Verify Mssql source data transfer Then Close the pipeline logs Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table - @MSSQL_SOURCE_TEST @BQ_SINK @Mssql_Required + @MSSQL_AS_SOURCE @BQ_AS_SINK Scenario: To verify the pipeline fails while preview with invalid bounding query setting the required split-By field Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWMacros.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWMacros.feature new file mode 100644 index 000000000..dbdc242a2 --- /dev/null +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWMacros.feature @@ -0,0 +1,111 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Mssql @Mssql_Required +Feature: Mssql Source - Run time scenarios W (macro) + + @MSSQL_AS_SOURCE @BQ_AS_SINK + Scenario: To verify data is getting transferred from Mssql to BigQuery successfully using macros for Connection section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "BigQuery" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "BigQuery" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqtarget.table" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "driverName" for key "DriverName" + Then Enter runtime argument value from environment variable "host" for key "Host" + Then Enter runtime argument value from environment variable "port" for key "Port" + Then Enter runtime argument value from environment variable "username" for key "Username" + Then Enter runtime argument value from environment variable "password" for key "Password" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + And Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value from environment variable "host" for key "Host" + And Enter runtime argument value from environment variable "port" for key "Port" + And Enter runtime argument value from environment variable "username" for key "Username" + And Enter runtime argument value from environment variable "password" for key "Password" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table + + @MSSQL_AS_SOURCE @BQ_AS_SINK + Scenario: To verify data is getting transferred from Mssql to BigQuery successfully using macros for Basic section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "BigQuery" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "BigQuery" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + And Click on the Macro button of Property: "database" and set the value to: "DatabaseName" + And Click on the Macro button of Property: "fetchSize" and set the value to: "fetchSize" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqtarget.table" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "databaseName" for key "DatabaseName" + Then Enter runtime argument value "fetchSize" for key "fetchSize" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "databaseName" for key "DatabaseName" + Then Enter runtime argument value "fetchSize" for key "fetchSize" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature index d48cd6bdd..b3df99d3d 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations under # the License. -@Mssql +@Mssql @Mssql_Required Feature: Mssql Source - Run time scenarios (macro) - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: To verify data is getting transferred from Mssql to Mssql successfully using macros for Connection section Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -69,7 +69,7 @@ Feature: Mssql Source - Run time scenarios (macro) Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table are equal to number of records from the source table - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET Scenario: To verify data is getting transferred from Mssql to Mssql successfully using macros for Basic section Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" @@ -118,176 +118,184 @@ Feature: Mssql Source - Run time scenarios (macro) Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table are equal to number of records from the source table - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required @BQ_SINK - Scenario: To verify data is getting transferred from Mssql to BigQuery successfully using macros for Connection section +# @MSSQL_AS_SOURCE @BQ_AS_SINK +# Scenario: To verify data is getting transferred from Mssql to BigQuery successfully using macros for Connection section +# Given Open Datafusion Project to configure pipeline +# When Expand Plugin group in the LHS plugins list: "Source" +# When Select plugin: "SQL Server" from the plugins list as: "Source" +# When Expand Plugin group in the LHS plugins list: "Sink" +# When Select plugin: "BigQuery" from the plugins list as: "Sink" +# Then Connect plugins: "SQL Server" and "BigQuery" to establish connection +# Then Navigate to the properties page of plugin: "SQL Server" +# And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" +# And Click on the Macro button of Property: "host" and set the value to: "Host" +# And Click on the Macro button of Property: "port" and set the value to: "Port" +# And Click on the Macro button of Property: "user" and set the value to: "Username" +# And Click on the Macro button of Property: "password" and set the value to: "Password" +# Then Enter input plugin property: "referenceName" with value: "sourceRef" +# Then Replace input plugin property: "database" with value: "databaseName" +# Then Enter textarea plugin property: "importQuery" with value: "selectQuery" +# Then Validate "SQL Server" plugin properties +# Then Close the Plugin Properties page +# And Navigate to the properties page of plugin: "BigQuery" +# And Enter input plugin property: "referenceName" with value: "Reference" +# And Replace input plugin property: "project" with value: "projectId" +# And Enter input plugin property: "datasetProject" with value: "datasetprojectId" +# And Enter input plugin property: "dataset" with value: "dataset" +# And Enter input plugin property: "table" with value: "bqtarget.table" +# Then Validate "BigQuery" plugin properties +# And Close the Plugin Properties page +# Then Save the pipeline +# Then Preview and run the pipeline +# Then Enter runtime argument value "driverName" for key "DriverName" +# Then Enter runtime argument value from environment variable "host" for key "Host" +# Then Enter runtime argument value from environment variable "port" for key "Port" +# Then Enter runtime argument value from environment variable "username" for key "Username" +# Then Enter runtime argument value from environment variable "password" for key "Password" +# And Run the preview of pipeline with runtime arguments +# Then Verify the preview of pipeline is "success" +# Then Close the preview +# Then Deploy the pipeline +# Then Run the Pipeline in Runtime +# And Enter runtime argument value "driverName" for key "DriverName" +# And Enter runtime argument value from environment variable "host" for key "Host" +# And Enter runtime argument value from environment variable "port" for key "Port" +# And Enter runtime argument value from environment variable "username" for key "Username" +# And Enter runtime argument value from environment variable "password" for key "Password" +# And Run the Pipeline in Runtime with runtime arguments +# Then Wait till pipeline is in running state +# Then Open and capture logs +# Then Verify the pipeline status is "Succeeded" +# Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table +# +# @MSSQL_AS_SOURCE @BQ_AS_SINK +# Scenario: To verify data is getting transferred from Mssql to BigQuery successfully using macros for Basic section +# Given Open Datafusion Project to configure pipeline +# When Expand Plugin group in the LHS plugins list: "Source" +# When Select plugin: "SQL Server" from the plugins list as: "Source" +# When Expand Plugin group in the LHS plugins list: "Sink" +# When Select plugin: "BigQuery" from the plugins list as: "Sink" +# Then Connect plugins: "SQL Server" and "BigQuery" to establish connection +# Then Navigate to the properties page of plugin: "SQL Server" +# Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" +# Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields +# Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields +# Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields +# Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields +# Then Enter input plugin property: "referenceName" with value: "sourceRef" +# And Click on the Macro button of Property: "database" and set the value to: "DatabaseName" +# And Click on the Macro button of Property: "fetchSize" and set the value to: "fetchSize" +# Then Enter textarea plugin property: "importQuery" with value: "selectQuery" +# Then Validate "SQL Server" plugin properties +# Then Close the Plugin Properties page +# And Navigate to the properties page of plugin: "BigQuery" +# And Enter input plugin property: "referenceName" with value: "Reference" +# And Replace input plugin property: "project" with value: "projectId" +# And Enter input plugin property: "datasetProject" with value: "datasetprojectId" +# And Enter input plugin property: "dataset" with value: "dataset" +# And Enter input plugin property: "table" with value: "bqtarget.table" +# Then Validate "BigQuery" plugin properties +# And Close the Plugin Properties page +# Then Save the pipeline +# Then Preview and run the pipeline +# Then Enter runtime argument value "databaseName" for key "DatabaseName" +# Then Enter runtime argument value "fetchSize" for key "fetchSize" +# And Run the preview of pipeline with runtime arguments +# Then Verify the preview of pipeline is "success" +# Then Close the preview +# Then Deploy the pipeline +# Then Run the Pipeline in Runtime +# Then Enter runtime argument value "databaseName" for key "DatabaseName" +# Then Enter runtime argument value "fetchSize" for key "fetchSize" +# And Run the Pipeline in Runtime with runtime arguments +# Then Wait till pipeline is in running state +# Then Open and capture logs +# Then Verify the pipeline status is "Succeeded" +# Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table + + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET + Scenario: Verify pipeline failure message in logs when user provides invalid Table name in importQuery of plugin with Macros Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" When Select plugin: "SQL Server" from the plugins list as: "Source" When Expand Plugin group in the LHS plugins list: "Sink" - When Select plugin: "BigQuery" from the plugins list as: "Sink" - Then Connect plugins: "SQL Server" and "BigQuery" to establish connection + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection Then Navigate to the properties page of plugin: "SQL Server" - And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" - And Click on the Macro button of Property: "host" and set the value to: "Host" - And Click on the Macro button of Property: "port" and set the value to: "Port" - And Click on the Macro button of Property: "user" and set the value to: "Username" - And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields Then Enter input plugin property: "referenceName" with value: "sourceRef" Then Replace input plugin property: "database" with value: "databaseName" - Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + And Click on the Macro button of Property: "importQuery" and set the value in textarea: "importQuery" Then Validate "SQL Server" plugin properties Then Close the Plugin Properties page - And Navigate to the properties page of plugin: "BigQuery" - And Enter input plugin property: "referenceName" with value: "Reference" - And Replace input plugin property: "project" with value: "projectId" - And Enter input plugin property: "datasetProject" with value: "datasetprojectId" - And Enter input plugin property: "dataset" with value: "dataset" - And Enter input plugin property: "table" with value: "bqtarget.table" - Then Validate "BigQuery" plugin properties - And Close the Plugin Properties page - Then Save the pipeline - Then Preview and run the pipeline - Then Enter runtime argument value "driverName" for key "DriverName" - Then Enter runtime argument value from environment variable "host" for key "Host" - Then Enter runtime argument value from environment variable "port" for key "Port" - Then Enter runtime argument value from environment variable "username" for key "Username" - Then Enter runtime argument value from environment variable "password" for key "Password" - And Run the preview of pipeline with runtime arguments - Then Verify the preview of pipeline is "success" - Then Close the preview - Then Deploy the pipeline - Then Run the Pipeline in Runtime - And Enter runtime argument value "driverName" for key "DriverName" - And Enter runtime argument value from environment variable "host" for key "Host" - And Enter runtime argument value from environment variable "port" for key "Port" - And Enter runtime argument value from environment variable "username" for key "Username" - And Enter runtime argument value from environment variable "password" for key "Password" + Then Navigate to the properties page of plugin: "SQL Server2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties + Then Close the Plugin Properties page + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Enter runtime argument value "invalidSelectQuery" for key "importQuery" And Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state - Then Open and capture logs - Then Verify the pipeline status is "Succeeded" - Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table + And Verify the pipeline status is "Failed" + Then Open Pipeline logs and verify Log entries having below listed Level and Message: + | Level | Message | + | ERROR | errorMessageInvalidsourcetable | - @MSSQL_SOURCE_TEST @Mssql_Required @BQ_SINK - Scenario: To verify data is getting transferred from Mssql to BigQuery successfully using macros for Basic section + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET + Scenario: Verify pipeline failure message in logs when user provides invalid Credentials for connection with Macros Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" When Select plugin: "SQL Server" from the plugins list as: "Source" When Expand Plugin group in the LHS plugins list: "Sink" - When Select plugin: "BigQuery" from the plugins list as: "Sink" - Then Connect plugins: "SQL Server" and "BigQuery" to establish connection + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection Then Navigate to the properties page of plugin: "SQL Server" + And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields - Then Enter input plugin property: "referenceName" with value: "sourceRef" - And Click on the Macro button of Property: "database" and set the value to: "DatabaseName" - And Click on the Macro button of Property: "fetchSize" and set the value to: "fetchSize" - Then Enter textarea plugin property: "importQuery" with value: "selectQuery" - Then Validate "SQL Server" plugin properties + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties Then Close the Plugin Properties page - Then Save the pipeline - Then Preview and run the pipeline - Then Enter runtime argument value "databaseName" for key "DatabaseName" - Then Enter runtime argument value "fetchSize" for key "fetchSize" - And Run the preview of pipeline with runtime arguments - Then Verify the preview of pipeline is "success" - Then Close the preview - Then Deploy the pipeline - Then Run the Pipeline in Runtime - Then Enter runtime argument value "databaseName" for key "DatabaseName" - Then Enter runtime argument value "fetchSize" for key "fetchSize" + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + Then Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value "invalid.host" for key "Host" + And Enter runtime argument value "invalid.port" for key "Port" + And Enter runtime argument value "invalid.username" for key "Username" + And Enter runtime argument value "invalid.password" for key "Password" And Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state - Then Open and capture logs - Then Verify the pipeline status is "Succeeded" - Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table - - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required - Scenario: Verify pipeline failure message in logs when user provides invalid Table name in importQuery of plugin with Macros - Given Open Datafusion Project to configure pipeline - When Expand Plugin group in the LHS plugins list: "Source" - When Select plugin: "SQL Server" from the plugins list as: "Source" - When Expand Plugin group in the LHS plugins list: "Sink" - When Select plugin: "SQL Server" from the plugins list as: "Sink" - Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection - Then Navigate to the properties page of plugin: "SQL Server" - Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" - Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields - Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields - Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields - Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields - Then Enter input plugin property: "referenceName" with value: "sourceRef" - Then Replace input plugin property: "database" with value: "databaseName" - And Click on the Macro button of Property: "importQuery" and set the value in textarea: "importQuery" - Then Validate "SQL Server" plugin properties - Then Close the Plugin Properties page - Then Navigate to the properties page of plugin: "SQL Server2" - Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" - Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields - Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields - Then Replace input plugin property: "database" with value: "databaseName" - Then Replace input plugin property: "tableName" with value: "targetTable" - Then Replace input plugin property: "dbSchemaName" with value: "schema" - Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields - Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields - Then Enter input plugin property: "referenceName" with value: "targetRef" - Then Validate "SQL Server2" plugin properties - Then Close the Plugin Properties page - And Save and Deploy Pipeline - And Run the Pipeline in Runtime - And Enter runtime argument value "invalidSelectQuery" for key "importQuery" - And Run the Pipeline in Runtime with runtime arguments - Then Wait till pipeline is in running state - And Verify the pipeline status is "Failed" - Then Open Pipeline logs and verify Log entries having below listed Level and Message: - | Level | Message | - | ERROR | errorMessageInvalidsourcetable | - - @MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required - Scenario: Verify pipeline failure message in logs when user provides invalid Credentials for connection with Macros - Given Open Datafusion Project to configure pipeline - When Expand Plugin group in the LHS plugins list: "Source" - When Select plugin: "SQL Server" from the plugins list as: "Source" - When Expand Plugin group in the LHS plugins list: "Sink" - When Select plugin: "SQL Server" from the plugins list as: "Sink" - Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection - Then Navigate to the properties page of plugin: "SQL Server" - And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" - And Click on the Macro button of Property: "host" and set the value to: "Host" - And Click on the Macro button of Property: "port" and set the value to: "Port" - And Click on the Macro button of Property: "user" and set the value to: "Username" - And Click on the Macro button of Property: "password" and set the value to: "Password" - Then Enter input plugin property: "referenceName" with value: "sourceRef" - Then Replace input plugin property: "database" with value: "databaseName" - Then Enter textarea plugin property: "importQuery" with value: "selectQuery" - Then Validate "SQL Server" plugin properties - Then Close the Plugin Properties page - Then Navigate to the properties page of plugin: "SQL Server2" - Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" - Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields - Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields - Then Replace input plugin property: "database" with value: "databaseName" - Then Replace input plugin property: "tableName" with value: "targetTable" - Then Replace input plugin property: "dbSchemaName" with value: "schema" - Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields - Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields - Then Enter input plugin property: "referenceName" with value: "targetRef" - Then Validate "SQL Server2" plugin properties - Then Close the Plugin Properties page - And Save and Deploy Pipeline - And Run the Pipeline in Runtime - Then Enter runtime argument value "driverName" for key "DriverName" - And Enter runtime argument value "invalid.host" for key "Host" - And Enter runtime argument value "invalid.port" for key "Port" - And Enter runtime argument value "invalid.username" for key "Username" - And Enter runtime argument value "invalid.password" for key "Password" - And Run the Pipeline in Runtime with runtime arguments - Then Wait till pipeline is in running state - And Verify the pipeline status is "Failed" - Then Open Pipeline logs and verify Log entries having below listed Level and Message: - | Level | Message | - | ERROR | errorMessageInvalidCredentialSource | + And Verify the pipeline status is "Failed" + Then Open Pipeline logs and verify Log entries having below listed Level and Message: + | Level | Message | + | ERROR | errorMessageInvalidCredentialSource | diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java index 499251bac..d0310dd95 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java @@ -36,7 +36,7 @@ */ public class MssqlClient { - private static Connection getMssqlConnection() throws SQLException, ClassNotFoundException { + public static Connection getMssqlConnection() throws SQLException, ClassNotFoundException { TimeZone timezone = TimeZone.getTimeZone("UTC"); TimeZone.setDefault(timezone); Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); @@ -142,6 +142,15 @@ public static void createSourceUniqueIdentifierTable(String sourceTable, String uniqueIdentifierValues)); } } + public static void createTargetMssqlTable(String targetTable, String schema) throws SQLException, + ClassNotFoundException { + try (Connection connect = getMssqlConnection(); + Statement statement = connect.createStatement()) { + String datatypesColumns = PluginPropertyUtils.pluginProp("SqlServerDatatypesColumns"); + String createSourceTableQuery2 = createTableQuery(targetTable, schema, datatypesColumns); + statement.executeUpdate(createSourceTableQuery2); + } + } public static void createTargetUniqueIdentifierTable(String targetTable, String schema) throws SQLException, ClassNotFoundException { @@ -176,13 +185,11 @@ public static void createTargetDateTimeTable(String targetTable, String schema) } } - public static void deleteTables(String schema, String[] tables) - throws SQLException, ClassNotFoundException { + public static void deleteTable(String schema, String table) + throws SQLException, ClassNotFoundException { try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) { - for (String table : tables) { String dropTableQuery = "DROP TABLE " + schema + "." + table; statement.execute(dropTableQuery); - } } } @@ -248,3 +255,4 @@ private static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarg return true; } } + diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java index 23a652ec4..624c3e3a7 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java @@ -39,8 +39,6 @@ * Mssql test hooks. */ public class TestSetupHooks { - public static String bqTargetTable = StringUtils.EMPTY; - @Before(order = 1) public static void setTableName() { String randomString = RandomStringUtils.randomAlphabetic(10).toUpperCase(); @@ -53,90 +51,186 @@ public static void setTableName() { sourceTableName)); } - @Before(order = 2, value = "@MSSQL_SOURCE_TEST") - public static void createTables() throws SQLException, ClassNotFoundException { + @Before(order = 2, value = "@MSSQL_AS_SOURCE") + public static void createSourceTables() throws SQLException, ClassNotFoundException { MssqlClient.createSourceTable(PluginPropertyUtils.pluginProp("sourceTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_AS_SOURCE") + public static void dropSourceTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("sourceTable")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " deleted successfully"); + } + + @Before(order = 2, value = "@MSSQL_AS_TARGET") + public static void createTargetTables() throws SQLException, ClassNotFoundException { MssqlClient.createTargetTable(PluginPropertyUtils.pluginProp("targetTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_AS_TARGET") + public static void dropTargetTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " deleted successfully"); } @Before(order = 2, value = "@MSSQL_TEST_TABLE") public static void createMssqlTestTable() throws SQLException, ClassNotFoundException { MssqlClient.createTargetMssqlTable(PluginPropertyUtils.pluginProp("targetTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " created successfully"); + } + @After(order = 2, value = "@MSSQL_TEST_TABLE") + public static void dropTestTargetTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " deleted successfully"); } @Before(order = 2, value = "@MSSQL_SOURCE_DATATYPES_TEST") - public static void createAllDatatypesTables() throws SQLException, ClassNotFoundException { + public static void createSourceDatatypesTables() throws SQLException, ClassNotFoundException { MssqlClient.createSourceDatatypesTable(PluginPropertyUtils.pluginProp("sourceTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_SOURCE_DATATYPES_TEST") + public static void dropSourceDatatypesTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("sourceTable")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " deleted successfully"); + } + + + @Before(order = 2, value = "@MSSQL_TARGET_DATATYPES_TEST") + public static void createTargetDatatypesTables() throws SQLException, ClassNotFoundException { MssqlClient.createTargetDatatypesTable(PluginPropertyUtils.pluginProp("targetTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL Target DataTypes Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_TARGET_DATATYPES_TEST") + public static void dropTargetDatatypesTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " deleted successfully"); } @Before(order = 2, value = "@MSSQL_SOURCE_DATATYPES_IMAGE_TEST") - public static void createDatatypesTablesImage() throws SQLException, ClassNotFoundException { + public static void createSourceDatatypesTablesImage() throws SQLException, ClassNotFoundException { MssqlClient.createSourceImageTable(PluginPropertyUtils.pluginProp("sourceTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_SOURCE_DATATYPES_IMAGE_TEST") + public static void dropSourceImageTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("sourceTable")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " deleted successfully"); + } + @Before(order = 2, value = "@MSSQL_TARGET_DATATYPES_IMAGE_TEST") + public static void createTargetDatatypesTablesImage() throws SQLException, ClassNotFoundException { MssqlClient.createTargetImageTable(PluginPropertyUtils.pluginProp("targetTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_TARGET_DATATYPES_IMAGE_TEST") + public static void dropTargetImageTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " deleted successfully"); } @Before(order = 2, value = "@MSSQL_SOURCE_DATATYPES_UIDTYPE_TEST") - public static void createDatatypesTablesUniqueIdentifier() throws SQLException, ClassNotFoundException { + public static void createSourceDatatypesTablesUniqueIdentifier() throws SQLException, ClassNotFoundException { MssqlClient.createSourceUniqueIdentifierTable(PluginPropertyUtils.pluginProp("sourceTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_SOURCE_DATATYPES_UIDTYPE_TEST") + public static void dropSourceDatatypesTablesUniqueIdentifier() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("sourceTable")); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " deleted successfully"); + } + + @Before(order = 2, value = "@MSSQL_TARGET_DATATYPES_UIDTYPE_TEST") + public static void createTargetDatatypesTablesUniqueIdentifier() throws SQLException, ClassNotFoundException { MssqlClient.createTargetUniqueIdentifierTable(PluginPropertyUtils.pluginProp("targetTable"), PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL UIDTYPE TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " created successfully"); + } + @After(order = 2, value = "@MSSQL_TARGET_DATATYPES_UIDTYPE_TEST") + public static void dropTargetDatatypesTablesUniqueIdentifier() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " deleted successfully"); } @Before(order = 2, value = "@MSSQL_SOURCE_DATATYPES_DATETIME_TEST") - public static void createDatatypesTablesDateTime() throws SQLException, ClassNotFoundException { + public static void createSourceDatatypesTablesDateTime() throws SQLException, ClassNotFoundException { MssqlClient.createSourceDateTimeTable(PluginPropertyUtils.pluginProp("sourceTable"), PluginPropertyUtils.pluginProp("schema")); - MssqlClient.createTargetDateTimeTable(PluginPropertyUtils.pluginProp("targetTable"), - PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL UIDTYPE SOURCE Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " created successfully"); + } - @After(order = 1, value = "@MSSQL_SINK_TEST") - public static void dropTables() throws SQLException, ClassNotFoundException { - MssqlClient.deleteTables(PluginPropertyUtils.pluginProp("schema"), - new String[]{PluginPropertyUtils.pluginProp("sourceTable"), - PluginPropertyUtils.pluginProp("targetTable")}); + @After(order = 2, value = "@MSSQL_SOURCE_DATATYPES_DATETIME_TEST") + public static void dropSourceDatatypesTablesDateTime() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("sourceTable")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("sourceTable") + " deleted successfully"); } - @Before(order = 1, value = "@BQ_SINK") - public static void setTempTargetBQTable() { - bqTargetTable = "TestSN_table" + UUID.randomUUID().toString().replaceAll("-", "_"); - PluginPropertyUtils.addPluginProp("bqtarget.table", bqTargetTable); - BeforeActions.scenario.write("BigQuery Target table name: " + bqTargetTable); + @Before(order = 2, value = "@MSSQL_TARGET_DATATYPES_DATETIME_TEST") + public static void createTargetDatatypesTablesDateTime() throws SQLException, ClassNotFoundException { + MssqlClient.createTargetDateTimeTable(PluginPropertyUtils.pluginProp("targetTable"), + PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("MSSQL UIDTYPE TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " created successfully"); } - @After(order = 1, value = "@BQ_SINK_CLEANUP") - public static void deleteTempTargetBQTable() throws IOException, InterruptedException { - try { - BigQueryClient.dropBqQuery(bqTargetTable); - BeforeActions.scenario.write("BigQuery Target table: " + bqTargetTable + " is deleted successfully"); - bqTargetTable = StringUtils.EMPTY; - } catch (BigQueryException e) { - if (e.getCode() == 404) { - BeforeActions.scenario.write("BigQuery Target Table: " + bqTargetTable + " does not exist"); - } else { - Assert.fail(e.getMessage()); - } - } + @After(order = 2, value = "@MSSQL_SOURCE_DATATYPES_DATETIME_TEST") + public static void dropTargetDatatypesTablesDateTime() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("MSSQL TARGET Table - " + + PluginPropertyUtils.pluginProp("targetTable") + " deleted successfully"); } - /** - * Create BigQuery table. - */ @Before(order = 1, value = "@BQ_SOURCE_TEST") public static void createTempSourceBQTable() throws IOException, InterruptedException { createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFile"), PluginPropertyUtils.pluginProp("InsertBQDataQueryFile")); } - @After(order = 1, value = "@BQ_SOURCE_TEST") public static void deleteTempSourceBQTable() throws IOException, InterruptedException { String bqSourceTable = PluginPropertyUtils.pluginProp("bqSourceTable"); @@ -150,6 +244,7 @@ private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile String bqSourceTable = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_"); + BeforeActions.scenario.write("Creating test data for Source BigQuery Table"); String createTableQuery = StringUtils.EMPTY; try { createTableQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource @@ -185,4 +280,27 @@ private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile PluginPropertyUtils.addPluginProp("bqSourceTable", bqSourceTable); BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " created successfully"); } + + @Before(order = 3, value = "@BQ_SINK_TEST") + public static void setTempTargetBQTable() { + String bqTargetTable = "TestSN_table" + RandomStringUtils.randomAlphanumeric(10); + PluginPropertyUtils.addPluginProp("bqtarget.table", bqTargetTable); + BeforeActions.scenario.write("BigQuery Target table name: " + bqTargetTable); + } + + @After(order = 1, value = "@BQ_SINK_TEST") + public static void deleteTempTargetBQTable() throws IOException, InterruptedException { + String bqTargetTable = PluginPropertyUtils.pluginProp("bqtarget.table"); + try { + BigQueryClient.dropBqQuery(bqTargetTable); + BeforeActions.scenario.write("BigQuery Target table: " + bqTargetTable + " is deleted successfully"); + bqTargetTable = StringUtils.EMPTY; + } catch (BigQueryException e) { + if (e.getCode() == 404) { + BeforeActions.scenario.write("BigQuery Target Table: " + bqTargetTable + " does not exist"); + } else { + Assert.fail(e.getMessage()); + } + } + } } diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/BQValidation.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/BQValidation.java index e78d2d94b..244066ede 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/BQValidation.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/BQValidation.java @@ -23,6 +23,7 @@ import io.cdap.plugin.MssqlClient; import org.apache.spark.sql.types.Decimal; import org.junit.Assert; +import stepsdesign.BeforeActions; import java.io.IOException; import java.sql.Connection; @@ -64,6 +65,9 @@ public static boolean validateDBToBQRecordValues(String schema, String sourceTab JsonObject json = new Gson().fromJson(String.valueOf(rows), JsonObject.class); bigQueryResponse.add(json); } + + BeforeActions.scenario.write("Big Query Data inside the validate method : " + bigQueryResponse); + String getSourceQuery = "SELECT * FROM " + schema + "." + sourceTable; try (Connection connect = MssqlClient.getMssqlConnection()) { connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); @@ -106,6 +110,9 @@ private static void getBigQueryTableData(String table, List bigQueryRows String selectQuery = "SELECT TO_JSON(t) FROM `" + projectId + "." + dataset + "." + table + "` AS t"; TableResult result = BigQueryClient.getQueryResult(selectQuery); result.iterateAll().forEach(value -> bigQueryRows.add(value.get(0).getValue())); + + BeforeActions.scenario.write("fetched the data from the BigQuery Table : " + bigQueryRows.size() + + bigQueryRows.toString()); } /** @@ -123,6 +130,8 @@ public static boolean compareResultSetAndJsonData(ResultSet rsSource, List 0) { columnCountTarget = bigQueryData.get(jsonObjectIdx).entrySet().size(); } + BeforeActions.scenario.write("BQ Columns Data" + columnCountTarget); + // Compare the number of columns in the source and target - Assert.assertEquals("Number of columns in source and target are not equal", - columnCountSource, columnCountTarget); + Assert.assertEquals(columnCountSource, columnCountTarget); while (rsSource.next()) { int currentColumnCount = 1; while (currentColumnCount <= columnCountSource) { diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/stepsdesign/Mssql.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/stepsdesign/Mssql.java index f1ece6c5e..9c4f0084d 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/stepsdesign/Mssql.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/stepsdesign/Mssql.java @@ -67,10 +67,13 @@ public void validateOUTRecordCountIsEqualToRecordsTransferredToTargetBigQueryTab @Then("Validate the values of records transferred to target BigQuery table is equal to the values from source Table") public void validateTheValuesOfRecordsTransferredToTargetBigQueryTableIsEqualToTheValuesFromSourceTable() throws InterruptedException, IOException, SQLException, ClassNotFoundException, ParseException { - int sourceBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqtarget.table")); - BeforeActions.scenario.write("No of Records from source BigQuery table:" + sourceBQRecordsCount); + int targetBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqtarget.table")); + + BeforeActions.scenario.write("No of Records in BigQuery table before going to data validation :" + + targetBQRecordsCount + "BQ Table :" + PluginPropertyUtils.pluginProp("bqtarget.table")); + Assert.assertEquals("Out records should match with target msSql table records count", - CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), sourceBQRecordsCount); + CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), targetBQRecordsCount); boolean recordsMatched = BQValidation.validateDBToBQRecordValues(PluginPropertyUtils.pluginProp("schema"), PluginPropertyUtils.pluginProp("sourceTable"), @@ -83,10 +86,10 @@ public void validateTheValuesOfRecordsTransferredToTargetBigQueryTableIsEqualToT "BigQuery table") public void validateTheValuesOfRecordsTransferredToTargetMsSqlTableIsEqualToTheValuesFromSourceBigQueryTable() throws InterruptedException, IOException, SQLException, ClassNotFoundException, ParseException { - int targetBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqSourceTable")); - BeforeActions.scenario.write("No of Records Transferred to BigQuery:" + targetBQRecordsCount); + int sourceBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqSourceTable")); + BeforeActions.scenario.write("No of Records Transferred to BigQuery:" + sourceBQRecordsCount); Assert.assertEquals("Out records should match with target BigQuery table records count", - CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), targetBQRecordsCount); + CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), sourceBQRecordsCount); boolean recordsMatched = BQValidation.validateBQToDBRecordValues(PluginPropertyUtils.pluginProp("schema"), PluginPropertyUtils.pluginProp("bqSourceTable"), PluginPropertyUtils.pluginProp("targetTable")); diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 25d5c90c0..00721f148 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -19,5 +19,5 @@ errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable 'connection' is invalid: Failed to assign value errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Incorrect syntax near the keyword 'table'.. \ Please check the system logs for more details. -errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Plugin with id SQL Server:\ - source.jdbc.sqlserver42 does not exist in program phase-1 of application +errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Plugin with id SQL \ + Server:source.jdbc.sqlserver does not exist in program phase-1 of application diff --git a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties index 11f62566e..9e1cbd46e 100644 --- a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -40,8 +40,10 @@ invalid.Port=MSSQL_PORT1 invalid.Username=MSSQL_USERNAME1 invalid.Password=MSSQL_PASSWORD1 #bq queries file path + CreateBQTableQueryFile=testData/BigQuery/BigQueryCreateTableQuery.txt InsertBQDataQueryFile=testData/BigQuery/BigQueryInsertDataQuery.txt + datatypeColumns=(ID VARCHAR(100) PRIMARY KEY, COL1 TINYINT, COL2 SMALLINT, COL3 INT, COL4 BIGINT,\ COL5 BIT, COL6 DECIMAL(38, 0), COL7 DECIMAL(10, 4), COL8 DECIMAL(5, 0), COL9 SMALLMONEY, COL10 MONEY, \ COL11 NUMERIC(38, 0), COL12 NUMERIC(10, 4), COL13 FLOAT, COL14 REAL, COL15 DATE, COL20 TIME(0), COL21 CHAR(100), \