From 4e185bd6a55e6cc7e9f928dd5228b9836ac61450 Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Defino Date: Thu, 29 Jun 2023 22:15:41 -0700 Subject: [PATCH 1/2] added covertura analysis through open clover and improved few tests --- README.md | 2 +- cloudbuild/cloudbuild.yaml | 16 +- cloudbuild/presubmit.sh | 13 +- flink-connector-bigquery/pom.xml | 56 ------ .../bigquery/fakes/StorageClientFaker.java | 17 +- .../split/BigQuerySourceSplitStateTest.java | 2 + .../BigQueryDynamicTableSourceITCase.java | 7 +- .../restrictions/BigQueryPartitionTest.java | 176 ++++++++++++++++++ pom.xml | 49 ++++- tools/maven/clover.xml | 45 +++++ 10 files changed, 292 insertions(+), 91 deletions(-) create mode 100644 flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/restrictions/BigQueryPartitionTest.java create mode 100644 tools/maven/clover.xml diff --git a/README.md b/README.md index ee0c5c62..c37d529c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Prerequisites: ``` git clone https://github.com/GoogleCloudDataproc/flink-bigquery-connector -cd flink-connector-bigquery +cd flink-bigquery-connector mvn clean package -DskipTests ``` diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml index 63db5a5e..d62996a8 100644 --- a/cloudbuild/cloudbuild.yaml +++ b/cloudbuild/cloudbuild.yaml @@ -13,27 +13,15 @@ steps: env: - 'CODECOV_TOKEN=${_CODECOV_TOKEN}' -# 3. Run unit tests +# 3. Run unit & integration tests - name: 'gcr.io/$PROJECT_ID/dataproc-flink-bigquery-connector-presubmit' id: 'unit-tests' waitFor: ['init'] entrypoint: 'bash' - args: ['/workspace/cloudbuild/presubmit.sh', 'unittest'] + args: ['/workspace/cloudbuild/presubmit.sh', 'tests'] env: - 'CODECOV_TOKEN=${_CODECOV_TOKEN}' -# 4. Run integration tests concurrently with unit tests -# Commeneted out until integration tests are ported -# - name: 'gcr.io/$PROJECT_ID/dataproc-flink-bigquery-connector-presubmit' -# id: 'integration-tests' -# waitFor: ['unit-tests'] -# entrypoint: 'bash' -# args: ['/workspace/cloudbuild/presubmit.sh', 'integrationtest'] -# env: -# - 'GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT}' -# - 'TEMPORARY_GCS_BUCKET=${_TEMPORARY_GCS_BUCKET}' -# - 'CODECOV_TOKEN=${_CODECOV_TOKEN}' - # Tests take around 20 mins in general. timeout: 1800s diff --git a/cloudbuild/presubmit.sh b/cloudbuild/presubmit.sh index 021c6330..e651b3ee 100644 --- a/cloudbuild/presubmit.sh +++ b/cloudbuild/presubmit.sh @@ -30,18 +30,13 @@ cd /workspace case $STEP in # Download maven and all the dependencies init) - $MVN install -DskipTests + $MVN clean install -DskipTests exit ;; - # Run unit tests - unittest) - $MVN test jacoco:report jacoco:report-aggregate -Pcoverage - ;; - - # Run integration tests - integrationtest) - $MVN failsafe:integration-test failsafe:verify jacoco:report jacoco:report-aggregate -Pcoverage,integration + # Run unit & integration tests + tests) + $MVN mvn clean clover:setup verify clover:aggregate clover:clover -Pclover -pl flink-connector-bigquery ;; *) diff --git a/flink-connector-bigquery/pom.xml b/flink-connector-bigquery/pom.xml index 1e90c672..cfcd2d82 100644 --- a/flink-connector-bigquery/pom.xml +++ b/flink-connector-bigquery/pom.xml @@ -162,66 +162,10 @@ under the License. org.apache.maven.plugins maven-jar-plugin - - - - test-jar - - - org.apache.maven.plugins maven-surefire-plugin - - - default-test - test - - test - - - ${argLine} -XX:+UseG1GC -Xms256m -Xmx1024m - - - - integration-tests - integration-test - - test - - - -XX:+UseG1GC -Xms256m -Xmx2048m - - - - - - org.jacoco - jacoco-maven-plugin - - - **/com/google/cloud/flink/bigquery/common/config/* - **/com/google/cloud/flink/bigquery/common/utils/GoogleCredentialsSupplier.class - **/com/google/cloud/flink/bigquery/services/**Impl.class - **/com/google/cloud/flink/bigquery/source/config/* - - - - - prepare-agent - - prepare-agent - - - - report - install - - report - - - diff --git a/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/fakes/StorageClientFaker.java b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/fakes/StorageClientFaker.java index a22aa268..7f87723b 100644 --- a/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/fakes/StorageClientFaker.java +++ b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/fakes/StorageClientFaker.java @@ -52,6 +52,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -80,16 +82,18 @@ public StorageReadClient getStorageClient(CredentialsOptions readOptions) @Override public QueryDataClient getQueryDataClient(CredentialsOptions readOptions) { return new QueryDataClient() { + @Override public List retrieveTablePartitions( String project, String dataset, String table) { - return Lists.newArrayList("2023062811"); + return Lists.newArrayList( + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHH"))); } @Override public Optional> retrievePartitionColumnName( String project, String dataset, String table) { - return Optional.of(Tuple2.of("number", StandardSQLTypeName.INT64)); + return Optional.of(Tuple2.of("ts", StandardSQLTypeName.TIMESTAMP)); } @Override @@ -189,8 +193,9 @@ public void close() {} public static final String SIMPLE_AVRO_SCHEMA_FIELDS_STRING = " \"fields\": [\n" - + " {\"name\": \"name\", \"type\": \"string\"},\n" - + " {\"name\": \"number\", \"type\": \"long\"}\n" + + " {\"name\": \"name\", \"type\": \"string\"},\n" + + " {\"name\": \"number\", \"type\": \"long\"},\n" + + " {\"name\" : \"ts\", \"type\" : {\"type\" : \"long\",\"logicalType\" : \"timestamp-micros\"}}\n" + " ]\n"; public static final String SIMPLE_AVRO_SCHEMA_STRING = "{\"namespace\": \"project.dataset\",\n" @@ -221,6 +226,10 @@ public void close() {} new TableFieldSchema() .setName("number") .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName("ts") + .setType("TIMESTAMP") .setMode("REQUIRED"))); /** Represents the parameters needed for the Avro data generation. */ diff --git a/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/source/split/BigQuerySourceSplitStateTest.java b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/source/split/BigQuerySourceSplitStateTest.java index c9725034..c03d42e6 100644 --- a/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/source/split/BigQuerySourceSplitStateTest.java +++ b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/source/split/BigQuerySourceSplitStateTest.java @@ -31,6 +31,8 @@ public void testSplitStateTransformation() { BigQuerySourceSplitState splitState = new BigQuerySourceSplitState(originalSplit); Assertions.assertThat(splitState.toBigQuerySourceSplit()).isEqualTo(originalSplit); + Assertions.assertThat(splitState) + .isEqualTo(new BigQuerySourceSplitState(splitState.toBigQuerySourceSplit())); } @Test diff --git a/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/BigQueryDynamicTableSourceITCase.java b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/BigQueryDynamicTableSourceITCase.java index 69c1ad38..f3832f39 100644 --- a/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/BigQueryDynamicTableSourceITCase.java +++ b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/BigQueryDynamicTableSourceITCase.java @@ -211,7 +211,12 @@ public void testProject() { @Test public void testRestriction() { - String sqlFilter = "id = 0 AND NOT optString IS NULL"; + String sqlFilter = + "id = 0" + + " AND NOT optString IS NULL" + + " AND optString LIKE 's%'" + + " AND optDouble > -1" + + " AND optDouble <= 1.0 "; tEnv.executeSql(createTestDDl(null)); Iterator collected = diff --git a/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/restrictions/BigQueryPartitionTest.java b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/restrictions/BigQueryPartitionTest.java new file mode 100644 index 00000000..23697c02 --- /dev/null +++ b/flink-connector-bigquery/src/test/java/com/google/cloud/flink/bigquery/table/restrictions/BigQueryPartitionTest.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.flink.bigquery.table.restrictions; + +import org.apache.flink.shaded.guava30.com.google.common.collect.Lists; + +import com.google.cloud.bigquery.StandardSQLTypeName; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import java.util.List; + +/** */ +public class BigQueryPartitionTest { + + @Test + public void testPartitionHour() { + List partitionIds = Lists.newArrayList("2023062822", "2023062823"); + // ISO formatted dates as single quote string literals at the beginning of the hour. + List expectedValues = + Lists.newArrayList("'2023-06-28 22:00:00'", "'2023-06-28 23:00:00'"); + List values = + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.TIMESTAMP); + + Assertions.assertThat(values).isEqualTo(expectedValues); + } + + @Test + public void testPartitionDay() { + List partitionIds = Lists.newArrayList("20230628", "20230628"); + // ISO formatted dates as single quote string literals. + List expectedValues = Lists.newArrayList("'2023-06-28'", "'2023-06-28'"); + List values = + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.DATETIME); + + Assertions.assertThat(values).isEqualTo(expectedValues); + } + + @Test + public void testPartitionMonth() { + List partitionIds = Lists.newArrayList("202306", "202307"); + // ISO formatted dates as single quote string literals + List expectedValues = Lists.newArrayList("'2023-06'", "'2023-07'"); + List values = + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.DATE); + + Assertions.assertThat(values).isEqualTo(expectedValues); + } + + @Test + public void testPartitionYear() { + List partitionIds = Lists.newArrayList("2023", "2022"); + // ISO formatted dates as single quote string literals + List expectedValues = Lists.newArrayList("'2023'", "'2022'"); + List values = + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.TIMESTAMP); + + Assertions.assertThat(values).isEqualTo(expectedValues); + } + + @Test + public void testPartitionInteger() { + List partitionIds = Lists.newArrayList("2023", "2022"); + // ISO formatted dates as single quote string literals + List expectedValues = Lists.newArrayList("2023", "2022"); + List values = + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.INT64); + + Assertions.assertThat(values).isEqualTo(expectedValues); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongTemporalPartition() { + List partitionIds = Lists.newArrayList("202308101112"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.TIMESTAMP); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongArrayPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType(partitionIds, StandardSQLTypeName.ARRAY); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongStructPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.STRUCT); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongJsonPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType(partitionIds, StandardSQLTypeName.JSON); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongGeoPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.GEOGRAPHY); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongBigNumPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.BIGNUMERIC); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongBoolPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType(partitionIds, StandardSQLTypeName.BOOL); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongBytesPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType(partitionIds, StandardSQLTypeName.BYTES); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongFloatPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.FLOAT64); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongStringPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.STRING); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongTimePartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType(partitionIds, StandardSQLTypeName.TIME); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongIntervalPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.INTERVAL); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongNumeriPartition() { + List partitionIds = Lists.newArrayList("2023", "2022"); + BigQueryPartition.partitionValuesFromIdAndDataType( + partitionIds, StandardSQLTypeName.NUMERIC); + } +} diff --git a/pom.xml b/pom.xml index 3d509416..f96c9146 100644 --- a/pom.xml +++ b/pom.xml @@ -56,12 +56,13 @@ under the License. false 3.0.0-1.16 - 0.8.10 + 4.4.1 1.7.36 2.17.2 flink-connector-bigquery-parent + target/test-report @@ -402,11 +403,47 @@ under the License. org.commonjava.maven.plugins directory-maven-plugin - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - + + + clover + + + + org.openclover + clover-maven-plugin + ${clover.version} + + + ${maven.multiModuleProjectDirectory}/tools/maven/clover.xml + 55% + + **/com/google/cloud/flink/bigquery/common/config/* + **/com/google/cloud/flink/bigquery/common/utils/GoogleCredentialsSupplier.* + **/com/google/cloud/flink/bigquery/services/**Impl.* + **/com/google/cloud/flink/bigquery/source/config/* + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${surefire.and.failsafe.report.dir} + + + + + + + + org.openclover + clover-maven-plugin + ${clover.version} + + + + + diff --git a/tools/maven/clover.xml b/tools/maven/clover.xml new file mode 100644 index 00000000..8149f176 --- /dev/null +++ b/tools/maven/clover.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 1dabd7083744b28bb38f8fc94dd0c69b2211003a Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Defino Date: Tue, 11 Jul 2023 16:17:01 -0700 Subject: [PATCH 2/2] removed repeated mvn reference from command --- cloudbuild/presubmit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/presubmit.sh b/cloudbuild/presubmit.sh index e651b3ee..71e7162d 100644 --- a/cloudbuild/presubmit.sh +++ b/cloudbuild/presubmit.sh @@ -36,7 +36,7 @@ case $STEP in # Run unit & integration tests tests) - $MVN mvn clean clover:setup verify clover:aggregate clover:clover -Pclover -pl flink-connector-bigquery + $MVN clean clover:setup verify clover:aggregate clover:clover -Pclover -pl flink-connector-bigquery ;; *)