Skip to content

Commit 594e570

Browse files
zabetakzabetak
authored
HIVE-24419: Refactor junit database rules to exploit testcontainers (#5928)
1 parent 12d88d9 commit 594e570

File tree

19 files changed

+304
-381
lines changed

19 files changed

+304
-381
lines changed

‎itests/hive-unit/pom.xml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
<name>Hive Integration - Unit Tests</name>
2626
<properties>
2727
<hive.path.to.root>../..</hive.path.to.root>
28-
<testcontainers.version>1.15.2</testcontainers.version>
2928
<htmlunit.version>2.70.0</htmlunit.version>
3029
</properties>
3130
<dependencies>
@@ -500,7 +499,6 @@
500499
<dependency>
501500
<groupId>org.testcontainers</groupId>
502501
<artifactId>testcontainers</artifactId>
503-
<version>${testcontainers.version}</version>
504502
<scope>test</scope>
505503
</dependency>
506504
<dependency>

‎itests/util/pom.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,26 @@
193193
<groupId>junit</groupId>
194194
<artifactId>junit</artifactId>
195195
</dependency>
196+
<dependency>
197+
<groupId>org.testcontainers</groupId>
198+
<artifactId>mariadb</artifactId>
199+
</dependency>
200+
<dependency>
201+
<groupId>org.testcontainers</groupId>
202+
<artifactId>mysql</artifactId>
203+
</dependency>
204+
<dependency>
205+
<groupId>org.testcontainers</groupId>
206+
<artifactId>postgresql</artifactId>
207+
</dependency>
208+
<dependency>
209+
<groupId>org.testcontainers</groupId>
210+
<artifactId>oracle-xe</artifactId>
211+
</dependency>
212+
<dependency>
213+
<groupId>org.testcontainers</groupId>
214+
<artifactId>mssqlserver</artifactId>
215+
</dependency>
196216
<dependency>
197217
<groupId>org.apache.hadoop</groupId>
198218
<artifactId>hadoop-hdfs</artifactId>

‎itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,10 @@ public Statement apply(final Statement base, Description description) {
7272
return new Statement() {
7373
@Override
7474
public void evaluate() throws Throwable {
75-
metaStoreHandler.setSystemProperties(); // for QTestUtil pre-initialization
76-
CliAdapter.this.beforeClass(); // instantiating QTestUtil
77-
7875
metaStoreHandler.getRule().before();
7976
metaStoreHandler.getRule().install();
77+
metaStoreHandler.setSystemProperties(); // for QTestUtil pre-initialization
78+
CliAdapter.this.beforeClass(); // instantiating QTestUtil
8079

8180
if (getQt() != null) {
8281
metaStoreHandler.setMetaStoreConfiguration(getQt().getConf());

‎itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java‎

Lines changed: 7 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,7 @@
2121
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
2222
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
2323
import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule;
24-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby;
25-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql;
26-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql;
27-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle;
28-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres;
29-
import org.apache.hadoop.hive.metastore.dbinstall.rules.PostgresTPCDS;
24+
import org.apache.hadoop.hive.metastore.dbinstall.rules.MetastoreRuleFactory;
3025
import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
3126
import org.slf4j.Logger;
3227
import org.slf4j.LoggerFactory;
@@ -46,7 +41,7 @@ public class QTestMetaStoreHandler {
4641
public QTestMetaStoreHandler(String metastore) {
4742
this.metastoreType = Objects.requireNonNull(metastore);
4843

49-
this.rule = getDatabaseRule(metastoreType).setVerbose(false);
44+
this.rule = MetastoreRuleFactory.create(metastoreType).setVerbose(false);
5045

5146
LOG.info(String.format("initialized metastore type '%s' for qtests", metastoreType));
5247
}
@@ -75,24 +70,6 @@ public QTestMetaStoreHandler setMetaStoreConfiguration(HiveConf conf) {
7570
return this;
7671
}
7772

78-
private DatabaseRule getDatabaseRule(String metastoreType) {
79-
switch (metastoreType) {
80-
case "postgres":
81-
return new Postgres();
82-
case "postgres.tpcds":
83-
return new PostgresTPCDS();
84-
case "oracle":
85-
return new Oracle();
86-
case "mysql":
87-
return new Mysql();
88-
case "mssql":
89-
case "sqlserver":
90-
return new Mssql();
91-
default:
92-
return new Derby();
93-
}
94-
}
95-
9673
private String getDbTypeConfString() {// "ORACLE", "MYSQL", "MSSQL", "POSTGRES"
9774
return "sqlserver".equalsIgnoreCase(metastoreType) ? "MSSQL" : metastoreType.toUpperCase();
9875
}
@@ -105,6 +82,11 @@ public void truncateDatabase(QTestUtil qt) throws Exception {
10582
}
10683
}
10784

85+
/**
86+
* Sets system properties for the metastore connection.
87+
* The metastore database must be initialized before calling this method. Some information, such as port
88+
* number (in JDBC URL), is not available until the rule is initialized.
89+
*/
10890
public void setSystemProperties() {
10991
System.setProperty(MetastoreConf.ConfVars.CONNECT_URL_KEY.getVarname(), rule.getJdbcUrl());
11092
System.setProperty(MetastoreConf.ConfVars.CONNECTION_DRIVER.getVarname(), rule.getJdbcDriver());

‎pom.xml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@
209209
<tez.version>0.10.5</tez.version>
210210
<super-csv.version>2.2.0</super-csv.version>
211211
<tempus-fugit.version>1.1</tempus-fugit.version>
212+
<testcontainers.version>1.21.3</testcontainers.version>
212213
<snappy.version>1.1.10.4</snappy.version>
213214
<jersey-wadl-doclet.version>4.0.0-M2</jersey-wadl-doclet.version>
214215
<jaxb-runtime.version>4.0.0-M2</jaxb-runtime.version>
@@ -1457,6 +1458,36 @@
14571458
<version>${postgres.version}</version>
14581459
<scope>runtime</scope>
14591460
</dependency>
1461+
<dependency>
1462+
<groupId>org.testcontainers</groupId>
1463+
<artifactId>testcontainers</artifactId>
1464+
<version>${testcontainers.version}</version>
1465+
</dependency>
1466+
<dependency>
1467+
<groupId>org.testcontainers</groupId>
1468+
<artifactId>mariadb</artifactId>
1469+
<version>${testcontainers.version}</version>
1470+
</dependency>
1471+
<dependency>
1472+
<groupId>org.testcontainers</groupId>
1473+
<artifactId>mysql</artifactId>
1474+
<version>${testcontainers.version}</version>
1475+
</dependency>
1476+
<dependency>
1477+
<groupId>org.testcontainers</groupId>
1478+
<artifactId>postgresql</artifactId>
1479+
<version>${testcontainers.version}</version>
1480+
</dependency>
1481+
<dependency>
1482+
<groupId>org.testcontainers</groupId>
1483+
<artifactId>oracle-xe</artifactId>
1484+
<version>${testcontainers.version}</version>
1485+
</dependency>
1486+
<dependency>
1487+
<groupId>org.testcontainers</groupId>
1488+
<artifactId>mssqlserver</artifactId>
1489+
<version>${testcontainers.version}</version>
1490+
</dependency>
14601491
</dependencies>
14611492
</dependencyManagement>
14621493
<dependencies>

‎ql/pom.xml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,31 @@
574574
<classifier>tests</classifier>
575575
<scope>test</scope>
576576
</dependency>
577+
<dependency>
578+
<groupId>org.testcontainers</groupId>
579+
<artifactId>mariadb</artifactId>
580+
<scope>test</scope>
581+
</dependency>
582+
<dependency>
583+
<groupId>org.testcontainers</groupId>
584+
<artifactId>mysql</artifactId>
585+
<scope>test</scope>
586+
</dependency>
587+
<dependency>
588+
<groupId>org.postgresql</groupId>
589+
<artifactId>postgresql</artifactId>
590+
<scope>test</scope>
591+
</dependency>
592+
<dependency>
593+
<groupId>org.testcontainers</groupId>
594+
<artifactId>oracle-xe</artifactId>
595+
<scope>test</scope>
596+
</dependency>
597+
<dependency>
598+
<groupId>org.testcontainers</groupId>
599+
<artifactId>mssqlserver</artifactId>
600+
<scope>test</scope>
601+
</dependency>
577602
<!-- test inter-project -->
578603
<dependency>
579604
<groupId>org.apache.parquet</groupId>

‎ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,7 @@
2121
import org.apache.hadoop.hive.metastore.DatabaseProduct;
2222
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
2323
import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule;
24-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby;
25-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Mariadb;
26-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql;
27-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql;
28-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle;
29-
import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres;
24+
import org.apache.hadoop.hive.metastore.dbinstall.rules.MetastoreRuleFactory;
3025
import org.junit.AfterClass;
3126
import org.junit.BeforeClass;
3227
import org.slf4j.Logger;
@@ -48,7 +43,10 @@ public static void setupDb() throws Exception {
4843
String metastoreType =
4944
System.getProperty(SYS_PROP_METASTORE_DB) == null ? "derby" : System.getProperty(SYS_PROP_METASTORE_DB)
5045
.toLowerCase();
51-
rule = getDatabaseRule(metastoreType).setVerbose(false);
46+
rule = MetastoreRuleFactory.create(metastoreType).setVerbose(false);
47+
// Start the docker container and create the hive user
48+
rule.before();
49+
rule.install();
5250

5351
conf.setVar(HiveConf.ConfVars.METASTORE_DB_TYPE, metastoreType.toUpperCase());
5452

@@ -62,30 +60,10 @@ public static void setupDb() throws Exception {
6260

6361
LOG.info("Set metastore connection to url: {}",
6462
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY));
65-
// Start the docker container and create the hive user
66-
rule.before();
67-
rule.install();
6863
}
6964

7065
@AfterClass
7166
public static void tearDownDb() {
7267
rule.after();
7368
}
74-
75-
private static DatabaseRule getDatabaseRule(String metastoreType) {
76-
switch (metastoreType) {
77-
case "postgres":
78-
return new Postgres();
79-
case "oracle":
80-
return new Oracle();
81-
case "mysql":
82-
return new Mysql();
83-
case "mariadb":
84-
return new Mariadb();
85-
case "mssql":
86-
return new Mssql();
87-
default:
88-
return new Derby();
89-
}
90-
}
9169
}

‎standalone-metastore/metastore-server/pom.xml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,31 @@
435435
<version>2.1.8</version>
436436
<scope>test</scope>
437437
</dependency>
438+
<dependency>
439+
<groupId>org.testcontainers</groupId>
440+
<artifactId>mariadb</artifactId>
441+
<scope>test</scope>
442+
</dependency>
443+
<dependency>
444+
<groupId>org.testcontainers</groupId>
445+
<artifactId>mysql</artifactId>
446+
<scope>test</scope>
447+
</dependency>
448+
<dependency>
449+
<groupId>org.testcontainers</groupId>
450+
<artifactId>postgresql</artifactId>
451+
<scope>test</scope>
452+
</dependency>
453+
<dependency>
454+
<groupId>org.testcontainers</groupId>
455+
<artifactId>oracle-xe</artifactId>
456+
<scope>test</scope>
457+
</dependency>
458+
<dependency>
459+
<groupId>org.testcontainers</groupId>
460+
<artifactId>mssqlserver</artifactId>
461+
<scope>test</scope>
462+
</dependency>
438463
</dependencies>
439464
<profiles>
440465
<profile>

0 commit comments

Comments
 (0)