Skip to content

Commit 0400a78

Browse files
zixi0825zixi0825
authored
[Refactor][Connector] Refactor connector dialect (#501) (#502)
1 parent fd024f4 commit 0400a78

File tree

109 files changed

+1070
-350
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

109 files changed

+1070
-350
lines changed

‎datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public interface ConnectorFactory {
2929

3030
Dialect getDialect();
3131

32-
ConnectorParameterConverter getConnectorParameterConverter();
32+
ParameterConverter getConnectorParameterConverter();
3333

3434
Executor getExecutor();
3535

@@ -42,4 +42,10 @@ public interface ConnectorFactory {
4242
StatementSplitter getStatementSplitter();
4343

4444
StatementParser getStatementParser();
45+
46+
MetricScript getMetricScript();
47+
48+
default Boolean showInFrontend() {
49+
return true;
50+
}
4551
}

‎datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/MetricScript.java

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package io.datavines.connector.api;
18+
19+
public interface MetricScript {
20+
21+
default String selectFromTable() {
22+
return "select * from ${table}";
23+
}
24+
25+
default String baseActualValue(String uniqueKey) {
26+
return "select count(1) as actual_value_"+ uniqueKey +" from ${invalidate_items_table}";
27+
}
28+
29+
default String baseDirectActualValue(String uniqueKey, String invalidateItemsSql) {
30+
return "select count(1) as actual_value_"+ uniqueKey +" from ( " + invalidateItemsSql + " ) t";
31+
}
32+
33+
default String avgActualValue(String uniqueKey) {
34+
return "select avg(${column}) as actual_value_"+ uniqueKey +" from ${table}";
35+
}
36+
37+
default String avgLengthActualValue(String uniqueKey) {
38+
return "select avg(length(${column})) as actual_value_"+ uniqueKey +" from ${table}";
39+
}
40+
41+
default String countDistinctActualValue(String uniqueKey) {
42+
return "select count(distinct(${column})) as actual_value_"+ uniqueKey +" from ${table}";
43+
}
44+
45+
default String histogramActualValue(String uniqueKey, String where) {
46+
return "select concat(k, '\001', cast(count as varchar)) as actual_value_" + uniqueKey + " from (select if(${column} is null, 'NULL', cast(${column} as varchar)) as k, count(1) as count from ${table} " + where + " group by ${column} order by count desc limit 50) T";
47+
}
48+
49+
default String maxActualValue(String uniqueKey) {
50+
return "select max(${column}) as actual_value_"+ uniqueKey +" from ${table}";
51+
}
52+
53+
default String maxLengthActualValue(String uniqueKey) {
54+
return "select max(length(${column})) as actual_value_"+ uniqueKey +" from ${table}";
55+
}
56+
57+
default String minActualValue(String uniqueKey) {
58+
return "select min(${column}) as actual_value_"+ uniqueKey +" from ${table}";
59+
}
60+
61+
default String minLengthActualValue(String uniqueKey) {
62+
return "select min(length(${column})) as actual_value_"+ uniqueKey +" from ${table}";
63+
}
64+
65+
default String stdDevActualValue(String uniqueKey) {
66+
return "select stddev(${column}) as actual_value_"+ uniqueKey +" from ${table}";
67+
}
68+
69+
default String sumActualValue(String uniqueKey) {
70+
return "select sum(${column}) as actual_value_"+ uniqueKey +" from ${table}";
71+
}
72+
73+
default String varianceActualValue(String uniqueKey) {
74+
return "select variance(${column}) as actual_value_"+ uniqueKey +" from ${table}";
75+
}
76+
77+
default String groupByHavingCountForUnique() {
78+
return " group by ${column} having count(1) = 1 ";
79+
}
80+
81+
default String groupByHavingCountForDuplicate() {
82+
return " group by ${column} having count(${column}) > 1 ";
83+
}
84+
85+
default String columnInEnums() {
86+
return " (${column} in ( ${enum_list} )) ";
87+
}
88+
89+
default String columnNotInEnums() {
90+
return " (${column} not in ( ${enum_list} ) or ${column} is null) ";
91+
}
92+
93+
default String columnNotNull() {
94+
return " ${column} is not null ";
95+
}
96+
97+
default String columnIsNull() {
98+
return " (${column} is null) ";
99+
}
100+
101+
default String columnLengthCompare() {
102+
return " length(${column}) ${comparator} ${length} ";
103+
}
104+
105+
default String columnNotMatchRegex() {
106+
return " ${column} not regexp '${regexp}' ";
107+
}
108+
109+
default String columnMatchRegex() {
110+
return " ${column} regexp '${regexp}' ";
111+
}
112+
113+
default String columnGteMin() {
114+
return " ${column} >= ${min} ";
115+
}
116+
117+
default String columnLteMax() {
118+
return " ${column} <= ${max} ";
119+
}
120+
121+
default String columnIsBlank() {
122+
return " (${column} is null or ${column} = '') ";
123+
}
124+
125+
default String timeBetweenWithFormat() {
126+
return " (DATE_FORMAT(${column}, '${datetime_format}') <= DATE_FORMAT(${deadline_time}, '${datetime_format}') ) AND (DATE_FORMAT(${column}, '${datetime_format}') >= DATE_FORMAT(${begin_time}, '${datetime_format}')) ";
127+
}
128+
129+
default String dailyAvg(String uniqueKey) {
130+
return "select round(avg(actual_value),2) as expected_value_" + uniqueKey +
131+
" from dv_actual_values where data_time >=date_format(${data_time},'%Y-%m-%d')" +
132+
" and data_time < date_add(date_format(${data_time},'%Y-%m-%d'), interval 1 DAY)" +
133+
" and unique_code = ${unique_code}";
134+
}
135+
136+
default String last7DayAvg(String uniqueKey) {
137+
return "select round(avg(actual_value),2) as expected_value_" + uniqueKey +
138+
" from dv_actual_values where data_time >= date_sub(date_format(${data_time},'%Y-%m-%d'),interval 7 DAY)" +
139+
" and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}";
140+
}
141+
142+
default String last30DayAvg(String uniqueKey) {
143+
return "select round(avg(actual_value),2) as expected_value_" + uniqueKey +
144+
" from dv_actual_values where data_time >= date_sub(date_format(${data_time},'%Y-%m-%d'),interval 30 DAY)" +
145+
" and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}";
146+
}
147+
148+
default String monthlyAvg(String uniqueKey) {
149+
return "select round(avg(actual_value),2) as expected_value_" + uniqueKey +
150+
" from dv_actual_values where data_time >= date_format(curdate(), '%Y-%m-01') and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY)" +
151+
" and unique_code = ${unique_code}";
152+
}
153+
154+
default String weeklyAvg(String uniqueKey) {
155+
return "select round(avg(actual_value),2) as expected_value_" + uniqueKey +
156+
" from dv_actual_values where data_time >= date_sub(${data_time},interval weekday(${data_time}) + 0 day)" +
157+
" and data_time < date_add(date_format(${data_time},'%Y-%m-%d'),interval 1 DAY) and unique_code = ${unique_code}";
158+
}
159+
}

‎datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorParameterConverter.java renamed to ‎datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ParameterConverter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
import static io.datavines.common.ConfigConstants.*;
2424

25-
public interface ConnectorParameterConverter {
25+
public interface ParameterConverter {
2626

2727
Map<String,Object> converter(Map<String,Object> parameter);
2828

‎datavines-connector/datavines-connector-plugins/datavines-connector-all/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,12 @@
158158
<version>${project.version}</version>
159159
</dependency>
160160

161+
<dependency>
162+
<groupId>io.datavines</groupId>
163+
<artifactId>datavines-connector-spark</artifactId>
164+
<version>${project.version}</version>
165+
</dependency>
166+
161167
</dependencies>
162168

163169
</project>

‎datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,13 @@
1616
*/
1717
package io.datavines.connector.plugin;
1818

19-
import io.datavines.connector.api.Connector;
20-
import io.datavines.connector.api.ConnectorParameterConverter;
21-
import io.datavines.connector.api.Dialect;
22-
import io.datavines.connector.api.Executor;
19+
import io.datavines.connector.api.*;
2320

2421
public class ClickHouseConnectorFactory extends AbstractJdbcConnectorFactory {
2522

2623
@Override
27-
public ConnectorParameterConverter getConnectorParameterConverter() {
28-
return new ClickHouseConnectorParameterConverter();
24+
public ParameterConverter getConnectorParameterConverter() {
25+
return new ClickHouseParameterConverter();
2926
}
3027

3128
@Override
@@ -42,4 +39,9 @@ public Connector getConnector() {
4239
public Executor getExecutor() {
4340
return new ClickHouseExecutor(getDataSourceClient());
4441
}
42+
43+
@Override
44+
public MetricScript getMetricScript() {
45+
return new ClickHouseMetricScript();
46+
}
4547
}

‎datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDialect.java

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,10 @@
1616
*/
1717
package io.datavines.connector.plugin;
1818

19-
import java.util.Map;
20-
21-
import static io.datavines.common.ConfigConstants.STD_DEV_KEY;
22-
import static io.datavines.common.ConfigConstants.VARIANCE_KEY;
23-
2419
public class ClickHouseDialect extends JdbcDialect {
2520

2621
@Override
2722
public String getDriver() {
2823
return "ru.yandex.clickhouse.ClickHouseDriver";
2924
}
30-
31-
@Override
32-
public Map<String, String> getDialectKeyMap() {
33-
super.getDialectKeyMap();
34-
dialectKeyMap.put(STD_DEV_KEY, "STDDEV_POP");
35-
dialectKeyMap.put(VARIANCE_KEY, "VAR_POP");
36-
return dialectKeyMap;
37-
38-
}
3925
}

‎datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseMetricScript.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package io.datavines.connector.plugin;
18+
19+
public class ClickHouseMetricScript extends JdbcMetricScript {
20+
21+
@Override
22+
public String stdDevActualValue(String uniqueKey) {
23+
return "select STDDEV_POP(${column}) as actual_value_"+ uniqueKey +" from ${table}";
24+
}
25+
26+
@Override
27+
public String varianceActualValue(String uniqueKey) {
28+
return "select VAR_POP(${column}) as actual_value_"+ uniqueKey +" from ${table}";
29+
}
30+
}

‎datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorParameterConverter.java renamed to ‎datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseParameterConverter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
import static io.datavines.common.ConfigConstants.*;
2222

23-
public class ClickHouseConnectorParameterConverter extends JdbcConnectorParameterConverter {
23+
public class ClickHouseParameterConverter extends JdbcParameterConverter {
2424

2525
@Override
2626
protected String getUrl(Map<String, Object> parameter) {

‎datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121
public class DatabendConnectorFactory extends AbstractJdbcConnectorFactory {
2222

2323
@Override
24-
public ConnectorParameterConverter getConnectorParameterConverter() {
25-
return new DatabendConnectorParameterConverter();
24+
public ParameterConverter getConnectorParameterConverter() {
25+
return new DatabendParameterConverter();
2626
}
2727

2828
@Override

‎datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorParameterConverter.java renamed to ‎datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendParameterConverter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
import static io.datavines.common.ConfigConstants.*;
2424

25-
public class DatabendConnectorParameterConverter extends JdbcConnectorParameterConverter {
25+
public class DatabendParameterConverter extends JdbcParameterConverter {
2626

2727
@Override
2828
protected String getUrl(Map<String, Object> parameter) {

‎datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ public Dialect getDialect() {
3131
}
3232

3333
@Override
34-
public ConnectorParameterConverter getConnectorParameterConverter() {
35-
return new DmConnectorParameterConverter();
34+
public ParameterConverter getConnectorParameterConverter() {
35+
return new DmParameterConverter();
3636
}
3737

3838
@Override

‎datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorParameterConverter.java renamed to ‎datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmParameterConverter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import static io.datavines.common.ConfigConstants.*;
2222
import static io.datavines.common.ConfigConstants.PROPERTIES;
2323

24-
public class DmConnectorParameterConverter extends JdbcConnectorParameterConverter {
24+
public class DmParameterConverter extends JdbcParameterConverter {
2525
@Override
2626
protected String getUrl(Map<String, Object> parameter) {
2727
// in dm jdbc url, the database is not need.

‎datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121
public class DorisConnectorFactory extends AbstractJdbcConnectorFactory {
2222

2323
@Override
24-
public ConnectorParameterConverter getConnectorParameterConverter() {
25-
return new DorisConnectorParameterConverter();
24+
public ParameterConverter getConnectorParameterConverter() {
25+
return new DorisParameterConverter();
2626
}
2727

2828
@Override

‎datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorParameterConverter.java renamed to ‎datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisParameterConverter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
import static io.datavines.common.ConfigConstants.*;
2424

25-
public class DorisConnectorParameterConverter extends MysqlConnectorParameterConverter {
25+
public class DorisParameterConverter extends MysqlParameterConverter {
2626

2727
@Override
2828
protected String getUrl(Map<String, Object> parameter) {

‎datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ public Dialect getDialect() {
4141
}
4242

4343
@Override
44-
public ConnectorParameterConverter getConnectorParameterConverter() {
45-
return new FileConnectorParameterConverter();
44+
public ParameterConverter getConnectorParameterConverter() {
45+
return new FileParameterConverter();
4646
}
4747

4848
@Override
@@ -74,4 +74,14 @@ public StatementSplitter getStatementSplitter() {
7474
public StatementParser getStatementParser() {
7575
return null;
7676
}
77+
78+
@Override
79+
public MetricScript getMetricScript() {
80+
return null;
81+
}
82+
83+
@Override
84+
public Boolean showInFrontend() {
85+
return false;
86+
}
7787
}

‎datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorParameterConverter.java renamed to ‎datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileParameterConverter.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
*/
1717
package io.datavines.connector.plugin;
1818

19-
import io.datavines.connector.api.ConnectorParameterConverter;
19+
import io.datavines.connector.api.ParameterConverter;
2020

2121
import java.util.Map;
2222

23-
public class FileConnectorParameterConverter implements ConnectorParameterConverter {
23+
public class FileParameterConverter implements ParameterConverter {
2424

2525
@Override
2626
public Map<String, Object> converter(Map<String, Object> parameter) {

0 commit comments

Comments
 (0)