Skip to content

Commit

Permalink
addressed comments for version range
Browse files Browse the repository at this point in the history
  • Loading branch information
@AnkitCLI
AnkitCLI committed Jan 11, 2024
1 parent 27906a9 commit 3ec8f59
Show file tree
Hide file tree
Showing 6 changed files with 25 additions and 63 deletions.
18 changes: 5 additions & 13 deletions ...c/e2e-test/resources/testData/Wrangler/BQ2BQwithWrnglerNGrpby-cdap-data-pipeline (1).json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -41,9 +41,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -80,9 +78,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "drop phonenumber\nuppercase :firstname\nset-type :zipcode integer \nfind-and-replace :streetAddress s/St./Street/Ig\nset-column :lastname_count string:length(lastname)\nfilter-rows-on regex-match lastname_count .*5.*\nfilter-rows-on condition-true zipcode == null || zipcode =~ \"^\\W*$\"",
Expand Down Expand Up @@ -123,9 +119,7 @@
"type": "batchaggregator",
"label": "Group By",
"artifact": {
"name": "core-plugins",
"version": "2.13.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "core-plugins"
},
"properties": {
"groupByFields": "city,firstname,lastname,state,zipcode,id",
Expand Down Expand Up @@ -162,9 +156,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...m/src/e2e-test/resources/testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-hl7 :Body\nhash :Body MD5\nset-type :Body string \nkeep address,Body,Body_hl7_MSH_12,Body_hl7_MSH_9_1\nfind-and-replace address s/address1/test/g\nmask-shuffle :Body_hl7_MSH_9_1\nsend-to-error empty(address)\nrename :Body_hl7_MSH_12 :id ",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...nsform/src/e2e-test/resources/testData/Wrangler/parse_csv_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-csv :body ',' false\nrename body_1 new_id\nquantize body_4 body_q 1:2=20,3:4=40\nset-type :body_4 integer \ncolumns-replace s/^new_//g\nfill-null-or-empty :body_3 'shubh'\nset-headers :abc\nchange-column-case uppercase\ncleanse-column-names\nsplit-to-rows :id '#'",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...m/src/e2e-test/resources/testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -257,9 +255,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-datetime :timestamp \"yyyy-MM-dd'T'HH:mm:ssX'['z']'\"\ncurrent-datetime :create_date\ndatetime-to-timestamp :timestamp\nformat-datetime :create_date 'y'\nformat-date :timestamp yyyy-mm-dd\nrename timestamp timecolumn",
Expand Down Expand Up @@ -301,9 +297,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...rc/e2e-test/resources/testData/Wrangler/parse_fixedlength_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-fixed-length :fixedlength 2,4,5,3\nsplit-url :Url\nwrite-as-csv :Url_protocol\nurl-encode :Url\nurl-decode :Url\nencode base32 :fixedlength\ndecode base32 :fixedlength_encode_base32\nsplit-to-columns :Url_query '='\nrename :fixedlength_2 :id\nfilter-rows-on condition-true fixedlength_4 !~ 'XYZ'",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions .../src/e2e-test/resources/testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-timestamp :time\nparse-as-currency :price :newprice\nformat-as-currency :newprice :format_price\nparse-as-simple-date :create_date yyyy-MM-dd\nparse-as-simple-date :update_date yyyy-MM-dd\ndiff-date :create_date :update_date :diff_date\ntimestamp-to-datetime :update_date\nrename :newprice :id",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down

0 comments on commit 3ec8f59

Please sign in to comment.