Skip to content

Commit

Permalink
Addressed comments
Browse files Browse the repository at this point in the history
  • Loading branch information
bharatgulati authored and AnkitCLI committed Aug 18, 2023
1 parent 07d8405 commit 32735a4
Show file tree
Hide file tree
Showing 22 changed files with 20 additions and 16 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
)
strategy:
matrix:
module: [wrangler-core]
module: [wrangler-transform]
fail-fast: false

steps:
Expand Down
19 changes: 13 additions & 6 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
<google.storage.version>1.106.0</google.storage.version>
<gson.version>2.6.2</gson.version>
<guava.retrying.version>2.0.0</guava.retrying.version>
<guava.version>20.0</guava.version>
<guava.version>31.0.1-jre</guava.version>
<hadoop.version>2.4.0</hadoop.version>
<hl7.version>2.2</hl7.version>
<hsql.version>2.2.4</hsql.version>
Expand Down Expand Up @@ -188,7 +188,7 @@
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<version>3.3.0</version>
<version>3.5.0</version>
<extensions>true</extensions>
<configuration>
<instructions>
Expand Down Expand Up @@ -234,7 +234,6 @@
<exclude>wrangler-demos/**</exclude>
<exclude>**/com/example/**</exclude>
<exclude>/**/icons/**</exclude>

</excludes>
</configuration>
</execution>
Expand Down Expand Up @@ -456,7 +455,14 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.0.0-M5</version>
<version>3.0.0</version>
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>3.0.0</version>
</dependency>
</dependencies>
<configuration>
<includes>
<include>${TEST_RUNNER}</include>
Expand Down Expand Up @@ -486,6 +492,7 @@
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
Expand Down Expand Up @@ -525,7 +532,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.0.1-jre</version>
<version>${guava.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
Expand All @@ -541,6 +548,7 @@
<groupId>io.cdap.tests.e2e</groupId>
<artifactId>cdap-e2e-framework</artifactId>
<version>0.3.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>

<dependency>
Expand All @@ -554,4 +562,3 @@
</profile>
</profiles>
</project>

1 change: 0 additions & 1 deletion wrangler-transform/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -215,5 +215,4 @@
</build>
</profile>
</profiles>

</project>
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Feature: Wrangler - Run time scenarios
Scenario: To verify User is able to run a pipeline using the copy count and delete directives in the wrangler plugin
Given Open Datafusion Project to configure pipeline
Then Click on the Plus Green Button to import the pipelines
Then Select the json files for importing the pipelines for the plugin "Directive_copy_drop_count_setcolmn"
Then Select the file for importing the pipeline for the plugin "Directive_copy_drop_count_setcolmn"
Then Navigate to the properties page of plugin: "BigQueryTable"
Then Replace input plugin property: "dataset" with value: "dataset"
Then Replace input plugin property: "table" with value: "bqSourceTable"
Expand All @@ -44,7 +44,7 @@ Feature: Wrangler - Run time scenarios
Scenario: To verify User is able to run a pipeline using the fill null and send to error directives in the wrangler plugin
Given Open Datafusion Project to configure pipeline
Then Click on the Plus Green Button to import the pipelines
Then Select the json files for importing the pipelines for the plugin "Directive_Fillempty_sendtoerror"
Then Select the file for importing the pipeline for the plugin "Directive_Fillempty_sendtoerror"
Then Navigate to the properties page of plugin: "BigQueryTable"
Then Replace input plugin property: "dataset" with value: "dataset"
Then Replace input plugin property: "table" with value: "bqSourceTable"
Expand All @@ -69,7 +69,7 @@ Feature: Wrangler - Run time scenarios
Scenario: To verify User is able to run a pipeline using the Format,concatenate,title case and copy column directives in the wrangler plugin
Given Open Datafusion Project to configure pipeline
Then Click on the Plus Green Button to import the pipelines
Then Select the json files for importing the pipelines for the plugin "Directive_Concatenate_titlecase"
Then Select the file for importing the pipeline for the plugin "Directive_Concatenate_titlecase"
Then Navigate to the properties page of plugin: "BigQueryTable"
Then Replace input plugin property: "dataset" with value: "dataset"
Then Replace input plugin property: "table" with value: "bqSourceTable"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,6 @@
import java.util.HashMap;
import java.util.Map;



/**
* Validation Helper.
*/
Expand All @@ -48,10 +46,10 @@ public static boolean validateActualDataToExpectedData(String table, String file
InterruptedException, URISyntaxException {
Map<String, JsonObject> bigQueryMap = new HashMap<>();
Map<String, JsonObject> fileMap = new HashMap<>();
Path importexpectedfile = Paths.get(ValidationHelper.class.getResource("/" + fileName).toURI());
Path importExpectedFile = Paths.get(ValidationHelper.class.getResource("/" + fileName).toURI());

getBigQueryTableData(table, bigQueryMap);
getFileData(importexpectedfile.toString(), fileMap);
getFileData(importExpectedFile.toString(), fileMap);

boolean isMatched = matchJsonMaps(bigQueryMap, fileMap);

Expand Down Expand Up @@ -95,7 +93,7 @@ private static void getBigQueryTableData(String targetTable, Map<String, JsonObj
String id = idElement.getAsString();
bigQueryMap.put(id, json);
} else {
Log.error("Data Mismatched");
LOG.error("Data Mismatched");
}
}
}
Expand Down

0 comments on commit 32735a4

Please sign in to comment.