From d94d0a904248a757ad7f1fc059b2d63bd729d221 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Sun, 14 Apr 2024 20:17:58 +0800 Subject: [PATCH 01/13] Add personalized configuration parameters for each metastore. --- ...ThriftHiveMetastoreIfaceClientFactory.java | 22 ++++++++++++++----- .../bdp/waggledance/context/CommonBeans.java | 1 + 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index ca62e4063..9911ba2ed 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -17,12 +17,10 @@ import static com.hotels.bdp.waggledance.api.model.ConnectionType.TUNNELED; -import java.util.Collections; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; +import com.hotels.bdp.waggledance.context.CommonBeans; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import lombok.AllArgsConstructor; @@ -45,7 +43,21 @@ public class CloseableThriftHiveMetastoreIfaceClientFactory { public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore) { Map properties = new HashMap<>(); if (waggleDanceConfiguration.getConfigurationProperties() != null) { - properties.putAll(waggleDanceConfiguration.getConfigurationProperties()); +// properties.putAll(waggleDanceConfiguration.getConfigurationProperties()); + Map serverConfigMap=waggleDanceConfiguration.getConfigurationProperties(); + Set keySet=serverConfigMap.keySet(); + + for(String key:keySet){ + if(!key.startsWith(CommonBeans.WD_HMS + ".")){ + properties.put(key,properties.get(key)); + } + } + String hmsPrefix = CommonBeans.WD_HMS + "." + metaStore.getName() + "."; + for(String key:keySet){ + if(key.startsWith(hmsPrefix)){ + properties.put(key.substring(hmsPrefix.length()),properties.get(key)); + } + } } return newHiveInstance(metaStore, properties); } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java index 98be2ca65..b1f0e9050 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java @@ -32,6 +32,7 @@ @org.springframework.context.annotation.Configuration public class CommonBeans { + public static final String WD_HMS = "wd-metastore"; @Bean public HiveConf hiveConf(WaggleDanceConfiguration waggleDanceConfiguration) { From 5b43c27b719f28b816e75734cd7b703e66d7c2e6 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Fri, 19 Apr 2024 17:19:56 +0800 Subject: [PATCH 02/13] Add personalized configuration parameters for each metastore --- .../api/model/AbstractMetaStore.java | 9 +++++++++ ...eThriftHiveMetastoreIfaceClientFactory.java | 18 +++--------------- .../bdp/waggledance/context/CommonBeans.java | 1 - 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index fdc91fa88..8fc7782d8 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -59,6 +59,7 @@ public abstract class AbstractMetaStore { private transient @JsonProperty @NotNull MetaStoreStatus status = MetaStoreStatus.UNKNOWN; private long latency = 0; private transient @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); + private Map configurationProperties = Collections.emptyMap(); public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) { this.name = name; @@ -243,4 +244,12 @@ public String toString() { .toString(); } + public Map getConfigurationProperties() { + return configurationProperties; + } + + public void setConfigurationProperties( + Map configurationProperties) { + this.configurationProperties = configurationProperties; + } } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index 9911ba2ed..a728180c1 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -43,22 +43,10 @@ public class CloseableThriftHiveMetastoreIfaceClientFactory { public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore) { Map properties = new HashMap<>(); if (waggleDanceConfiguration.getConfigurationProperties() != null) { -// properties.putAll(waggleDanceConfiguration.getConfigurationProperties()); - Map serverConfigMap=waggleDanceConfiguration.getConfigurationProperties(); - Set keySet=serverConfigMap.keySet(); - - for(String key:keySet){ - if(!key.startsWith(CommonBeans.WD_HMS + ".")){ - properties.put(key,properties.get(key)); - } - } - String hmsPrefix = CommonBeans.WD_HMS + "." + metaStore.getName() + "."; - for(String key:keySet){ - if(key.startsWith(hmsPrefix)){ - properties.put(key.substring(hmsPrefix.length()),properties.get(key)); - } - } + properties.putAll(waggleDanceConfiguration.getConfigurationProperties()); } + //override per metastore + properties.putAll(metaStore.getConfigurationProperties()); return newHiveInstance(metaStore, properties); } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java index b1f0e9050..98be2ca65 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java @@ -32,7 +32,6 @@ @org.springframework.context.annotation.Configuration public class CommonBeans { - public static final String WD_HMS = "wd-metastore"; @Bean public HiveConf hiveConf(WaggleDanceConfiguration waggleDanceConfiguration) { From 394250d049e448b520fd6a07f4326db113c551df Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Mon, 22 Apr 2024 10:44:12 +0800 Subject: [PATCH 03/13] Recover --- .../CloseableThriftHiveMetastoreIfaceClientFactory.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index a728180c1..c11570c08 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -17,10 +17,12 @@ import static com.hotels.bdp.waggledance.api.model.ConnectionType.TUNNELED; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; import java.util.concurrent.TimeUnit; -import com.hotels.bdp.waggledance.context.CommonBeans; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import lombok.AllArgsConstructor; From 14abc776e31f7d1b59a99af39915b7459a97cde9 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Sat, 11 May 2024 14:43:29 +0800 Subject: [PATCH 04/13] Update junit test --- .../bdp/waggledance/api/model/FederatedMetaStoreTest.java | 2 +- .../hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java index 86048bb53..dcbe47103 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java @@ -72,7 +72,7 @@ public void nullDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java index 8c1634eb6..1e1b1dc67 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java @@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); From 877a76a708f265d096682b2d141217129905dc73 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Sat, 11 May 2024 17:02:21 +0800 Subject: [PATCH 05/13] Update Junit Test --- .../api/model/AbstractMetaStore.java | 4 +- ...ThriftHiveMetastoreIfaceClientFactory.java | 4 +- .../YamlFederatedMetaStoreStorageTest.java | 85 ++++++++++--------- 3 files changed, 51 insertions(+), 42 deletions(-) diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index 8fc7782d8..600917f15 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -51,7 +51,7 @@ public abstract class AbstractMetaStore { private List writableDatabaseWhitelist; private List mappedDatabases; private @Valid List mappedTables; - private Map databaseNameMapping = Collections.emptyMap(); + private transient Map databaseNameMapping = Collections.emptyMap(); private @NotBlank String name; private @NotBlank String remoteMetaStoreUris; private @Valid MetastoreTunnel metastoreTunnel; @@ -59,7 +59,7 @@ public abstract class AbstractMetaStore { private transient @JsonProperty @NotNull MetaStoreStatus status = MetaStoreStatus.UNKNOWN; private long latency = 0; private transient @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); - private Map configurationProperties = Collections.emptyMap(); + private Map configurationProperties;// = Collections.emptyMap(); public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) { this.name = name; diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index c11570c08..90e907169 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -48,7 +48,9 @@ public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore properties.putAll(waggleDanceConfiguration.getConfigurationProperties()); } //override per metastore - properties.putAll(metaStore.getConfigurationProperties()); + if(metaStore.getConfigurationProperties() != null) { + properties.putAll(metaStore.getConfigurationProperties()); + } return newHiveInstance(metaStore, properties); } diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java index bb99938ae..fa1c88961 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.Collections; import java.util.List; import javax.validation.ConstraintViolationException; @@ -211,10 +212,11 @@ public void saveFederationWriteFederations() throws Exception { MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2")); newFederatedInstance.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2)); newFederatedInstance.setHiveMetastoreFilterHook("filter.hook.class"); + newFederatedInstance.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal","hive/_HOST@REALM")); storage.insert(newFederatedInstance); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(24)); + assertThat(lines.size(), is(26)); assertThat(lines.get(0), is("primary-meta-store:")); assertThat(lines.get(1), is(" access-control-type: READ_ONLY")); assertThat(lines.get(2), is(" database-prefix: ''")); @@ -223,22 +225,24 @@ public void saveFederationWriteFederations() throws Exception { assertThat(lines.get(5), is(" remote-meta-store-uris: thrift://localhost:19083")); assertThat(lines.get(6), is("federated-meta-stores:")); assertThat(lines.get(7), is("- access-control-type: READ_ONLY")); - assertThat(lines.get(8), is(" database-name-mapping: {}")); - assertThat(lines.get(9), is(" database-prefix: hcom_2_")); - assertThat(lines.get(10), is(" hive-metastore-filter-hook: filter.hook.class")); - assertThat(lines.get(11), is(" latency: 0")); - assertThat(lines.get(12), is(" mapped-databases:")); - assertThat(lines.get(13), is(" - db1")); - assertThat(lines.get(14), is(" - db2")); - assertThat(lines.get(15), is(" mapped-tables:")); - assertThat(lines.get(16), is(" - database: db1")); - assertThat(lines.get(17), is(" mapped-tables:")); - assertThat(lines.get(18), is(" - tbl1")); - assertThat(lines.get(19), is(" - database: db2")); - assertThat(lines.get(20), is(" mapped-tables:")); - assertThat(lines.get(21), is(" - tbl2")); - assertThat(lines.get(22), is(" name: hcom_2")); - assertThat(lines.get(23), is(" remote-meta-store-uris: thrift://localhost:29083")); + assertThat(lines.get(8), is(" configuration-properties:")); + assertThat(lines.get(9), is(" hive.metastore.kerberos.principal: hive/_HOST@REALM")); + assertThat(lines.get(10), is(" database-name-mapping: {}")); + assertThat(lines.get(11), is(" database-prefix: hcom_2_")); + assertThat(lines.get(12), is(" hive-metastore-filter-hook: filter.hook.class")); + assertThat(lines.get(13), is(" latency: 0")); + assertThat(lines.get(14), is(" mapped-databases:")); + assertThat(lines.get(15), is(" - db1")); + assertThat(lines.get(16), is(" - db2")); + assertThat(lines.get(17), is(" mapped-tables:")); + assertThat(lines.get(18), is(" - database: db1")); + assertThat(lines.get(19), is(" mapped-tables:")); + assertThat(lines.get(20), is(" - tbl1")); + assertThat(lines.get(21), is(" - database: db2")); + assertThat(lines.get(22), is(" mapped-tables:")); + assertThat(lines.get(23), is(" - tbl2")); + assertThat(lines.get(24), is(" name: hcom_2")); + assertThat(lines.get(25), is(" remote-meta-store-uris: thrift://localhost:29083")); } @Test @@ -292,34 +296,37 @@ public void savePrimaryWriteFederations() throws Exception { MappedTables mappedTables1 = new MappedTables("db1", Lists.newArrayList("tbl1")); MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2")); primaryMetaStore.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2)); + primaryMetaStore.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal","hive/_HOST@REALM")); storage.insert(primaryMetaStore); storage.insert(newFederatedInstance("hcom_2", "thrift://localhost:29083")); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(23)); + assertThat(lines.size(), is(25)); assertThat(lines.get(0), is("primary-meta-store:")); assertThat(lines.get(1), is(" access-control-type: READ_ONLY")); - assertThat(lines.get(2), is(" database-prefix: ''")); - assertThat(lines.get(3), is(" latency: 0")); - assertThat(lines.get(4), is(" mapped-databases:")); - assertThat(lines.get(5), is(" - db1")); - assertThat(lines.get(6), is(" - db2")); - assertThat(lines.get(7), is(" mapped-tables:")); - assertThat(lines.get(8), is(" - database: db1")); - assertThat(lines.get(9), is(" mapped-tables:")); - assertThat(lines.get(10), is(" - tbl1")); - assertThat(lines.get(11), is(" - database: db2")); - assertThat(lines.get(12), is(" mapped-tables:")); - assertThat(lines.get(13), is(" - tbl2")); - assertThat(lines.get(14), is(" name: hcom_1")); - assertThat(lines.get(15), is(" remote-meta-store-uris: thrift://localhost:19083")); - assertThat(lines.get(16), is("federated-meta-stores:")); - assertThat(lines.get(17), is("- access-control-type: READ_ONLY")); - assertThat(lines.get(18), is(" database-name-mapping: {}")); - assertThat(lines.get(19), is(" database-prefix: hcom_2_")); - assertThat(lines.get(20), is(" latency: 0")); - assertThat(lines.get(21), is(" name: hcom_2")); - assertThat(lines.get(22), is(" remote-meta-store-uris: thrift://localhost:29083")); + assertThat(lines.get(2), is(" configuration-properties:")); + assertThat(lines.get(3), is(" hive.metastore.kerberos.principal: hive/_HOST@REALM")); + assertThat(lines.get(4), is(" database-prefix: ''")); + assertThat(lines.get(5), is(" latency: 0")); + assertThat(lines.get(6), is(" mapped-databases:")); + assertThat(lines.get(7), is(" - db1")); + assertThat(lines.get(8), is(" - db2")); + assertThat(lines.get(9), is(" mapped-tables:")); + assertThat(lines.get(10), is(" - database: db1")); + assertThat(lines.get(11), is(" mapped-tables:")); + assertThat(lines.get(12), is(" - tbl1")); + assertThat(lines.get(13), is(" - database: db2")); + assertThat(lines.get(14), is(" mapped-tables:")); + assertThat(lines.get(15), is(" - tbl2")); + assertThat(lines.get(16), is(" name: hcom_1")); + assertThat(lines.get(17), is(" remote-meta-store-uris: thrift://localhost:19083")); + assertThat(lines.get(18), is("federated-meta-stores:")); + assertThat(lines.get(19), is("- access-control-type: READ_ONLY")); + assertThat(lines.get(20), is(" database-name-mapping: {}")); + assertThat(lines.get(21), is(" database-prefix: hcom_2_")); + assertThat(lines.get(22), is(" latency: 0")); + assertThat(lines.get(23), is(" name: hcom_2")); + assertThat(lines.get(24), is(" remote-meta-store-uris: thrift://localhost:29083")); } private PrimaryMetaStore newPrimaryInstance(String name, String remoteMetaStoreUris) { From 085d7fc78295245c6f0d71d908d66047e82100e4 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Sat, 11 May 2024 17:18:08 +0800 Subject: [PATCH 06/13] Update Junit Test --- .../bdp/waggledance/api/model/FederatedMetaStoreTest.java | 2 +- .../hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java index dcbe47103..0f4fdc8ac 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java @@ -72,7 +72,7 @@ public void nullDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":null,\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java index 1e1b1dc67..4a3208fc3 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java @@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":null,\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); From cb7b7bfcd339dd434c4cc51ad6f2ea8b2d8dcdb1 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Mon, 13 May 2024 09:40:14 +0800 Subject: [PATCH 07/13] Update Junit Test --- ...loseableThriftHiveMetastoreIfaceClientFactoryTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java index d011d50cd..9ef1d1f15 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java @@ -24,10 +24,12 @@ import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import com.hotels.bdp.waggledance.api.model.FederatedMetaStore; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.junit.Before; @@ -68,8 +70,9 @@ public void setUp() { @Test public void defaultFactory() { ArgumentCaptor hiveConfCaptor = ArgumentCaptor.forClass(HiveConf.class); - - factory.newInstance(newFederatedInstance("fed1", THRIFT_URI)); + FederatedMetaStore fed1 = newFederatedInstance("fed1", THRIFT_URI); + fed1.setConfigurationProperties(Collections.singletonMap(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname,"hive/_HOST@HADOOP.COM")); + factory.newInstance(fed1); verify(defaultMetaStoreClientFactory).newInstance(hiveConfCaptor.capture(), eq( "waggledance-fed1"), eq(3), eq(2000)); verifyNoInteractions(tunnelingMetaStoreClientFactory); @@ -80,6 +83,7 @@ public void defaultFactory() { assertThat(hiveConf.getTimeVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS), is(5L)); assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT), is(true)); assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_COMPACT_PROTOCOL), is(false)); + assertThat(hiveConf.getVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL), is("hive/_HOST@HADOOP.COM")); } @Test From 83074049bad44758bbef4a9ac0ac5fc75b2e8f5d Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Fri, 24 May 2024 17:45:07 +0800 Subject: [PATCH 08/13] Format the code and update the readme --- README.md | 57 ++++++++++--------- .../api/model/AbstractMetaStore.java | 18 +++--- ...ThriftHiveMetastoreIfaceClientFactory.java | 4 +- ...ftHiveMetastoreIfaceClientFactoryTest.java | 2 +- .../YamlFederatedMetaStoreStorageTest.java | 4 +- 5 files changed, 45 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 4757b33e2..926bcf775 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,8 @@ Example: - my_writable_db2 - user_db_.* - ... + configuration-properties: + hive.metastore.kerberos.principal: hive/_HOST@HADOOP.COM federated-meta-stores: # List of read only metastores to federate - remote-meta-store-uris: thrift://10.0.0.1:9083 name: secondary @@ -147,35 +149,38 @@ Example: - database: prod_db2 mapped-tables: - tbl2 + configuration-properties: + hive.metastore.kerberos.principal: hive/clustername@HADOOP.COM - ... - The table below describes all the available configuration values for Waggle Dance federations: -| Property | Required | Description | -|:----|:----:|:----| -| `primary-meta-store` | No | Primary MetaStore config. Can be empty but it is advised to configure it. | -| `primary-meta-store.remote-meta-store-uris` | Yes | Thrift URIs of the federated read-only metastore. | -| `primary-meta-store.name` | Yes | Database name that uniquely identifies this metastore. Used internally. Cannot be empty. | -| `primary-meta-store.database-prefix` | No | Prefix used to access the primary metastore and differentiate databases in it from databases in another metastore. The default prefix (i.e. if this value isn't explicitly set) is empty string.| -| `primary-meta-store.access-control-type` | No | Sets how the client access controls should be handled. Default is `READ_ONLY` Other options `READ_AND_WRITE_AND_CREATE`, `READ_AND_WRITE_ON_DATABASE_WHITELIST` and `READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST` see Access Control section below. | -| `primary-meta-store.writable-database-white-list` | No | White-list of databases used to verify write access used in conjunction with `primary-meta-store.access-control-type`. The list of databases should be listed without any `primary-meta-store.database-prefix`. This property supports both full database names and (case-insensitive) [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html).| -| `primary-meta-store.metastore-tunnel` | No | See metastore tunnel configuration values below. | -| `primary-meta-store.latency` | No | Indicates the acceptable slowness of the metastore in **milliseconds** for increasing the default connection timeout. Default latency is `0` and should be changed if the metastore is particularly slow. If you get an error saying that results were omitted because the metastore was slow, consider changing the latency to a higher number.| -| `primary-meta-store.mapped-databases` | No | List of databases to federate from the primary metastore; all other databases will be ignored. This property supports both full database names and [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) (both being case-insensitive). By default, all databases from the metastore are federated. | -| `primary-meta-store.mapped-tables` | No | List of mappings from databases to tables to federate from the primary metastore, similar to `mapped-databases`. By default, all tables are available. See `mapped-tables` configuration below. | -| `primary-meta-stores.hive-metastore-filter-hook` | No | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. | -| `primary-meta-stores.database-name-mapping` | No | BiDirectional Map of database names and mapped name, where key=`` and value=``. See the [Database Name Mapping](#database-name-mapping) section.| -| `federated-meta-stores` | No | Possible empty list of read only federated metastores. | -| `federated-meta-stores[n].remote-meta-store-uris` | Yes | Thrift URIs of the federated read-only metastore. | -| `federated-meta-stores[n].name` | Yes | Name that uniquely identifies this metastore. Used internally. Cannot be empty. | -| `federated-meta-stores[n].database-prefix` | No | Prefix used to access this particular metastore and differentiate databases in it from databases in another metastore. Typically used if databases have the same name across metastores but federated access to them is still needed. The default prefix (i.e. if this value isn't explicitly set) is {federated-meta-stores[n].name} lowercased and postfixed with an underscore. For example if the metastore name was configured as "waggle" and no database prefix was provided but `PREFIXED` database resolution was used then the value of `database-prefix` would be "waggle_". | -| `federated-meta-stores[n].metastore-tunnel` | No | See metastore tunnel configuration values below. | -| `federated-meta-stores[n].latency` | No | Indicates the acceptable slowness of the metastore in **milliseconds** for increasing the default connection timeout. Default latency is `0` and should be changed if the metastore is particularly slow. If you get an error saying that results were omitted because the metastore was slow, consider changing the latency to a higher number.| -| `federated-meta-stores[n].mapped-databases` | No | List of databases to federate from this federated metastore, all other databases will be ignored. This property supports both full database names and [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) (both being case-insensitive). By default, all databases from the metastore are federated. | -| `federated-meta-stores[n].mapped-tables` | No | List of mappings from databases to tables to federate from this federated metastore, similar to `mapped-databases`. By default, all tables are available. See `mapped-tables` configuration below. | -| `federated-meta-stores[n].hive-metastore-filter-hook` | No | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. | -| `federated-meta-stores[n].database-name-mapping` | No | BiDirectional Map of database names and mapped names where key=`` and value=``. See the [Database Name Mapping](#database-name-mapping) section.| -| `federated-meta-stores[n].writable-database-white-list` | No | White-list of databases used to verify write access used in conjunction with `federated-meta-stores[n].access-control-type`. The list of databases should be listed without a `federated-meta-stores[n].database-prefix`. This property supports both full database names and (case-insensitive) [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html).| +| Property | Required | Description | +|:----------------------------------------------------------|:----:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `primary-meta-store` | No | Primary MetaStore config. Can be empty but it is advised to configure it. | +| `primary-meta-store.remote-meta-store-uris` | Yes | Thrift URIs of the federated read-only metastore. | +| `primary-meta-store.name` | Yes | Database name that uniquely identifies this metastore. Used internally. Cannot be empty. | +| `primary-meta-store.database-prefix` | No | Prefix used to access the primary metastore and differentiate databases in it from databases in another metastore. The default prefix (i.e. if this value isn't explicitly set) is empty string. | +| `primary-meta-store.access-control-type` | No | Sets how the client access controls should be handled. Default is `READ_ONLY` Other options `READ_AND_WRITE_AND_CREATE`, `READ_AND_WRITE_ON_DATABASE_WHITELIST` and `READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST` see Access Control section below. | +| `primary-meta-store.writable-database-white-list` | No | White-list of databases used to verify write access used in conjunction with `primary-meta-store.access-control-type`. The list of databases should be listed without any `primary-meta-store.database-prefix`. This property supports both full database names and (case-insensitive) [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html). | +| `primary-meta-store.metastore-tunnel` | No | See metastore tunnel configuration values below. | +| `primary-meta-store.latency` | No | Indicates the acceptable slowness of the metastore in **milliseconds** for increasing the default connection timeout. Default latency is `0` and should be changed if the metastore is particularly slow. If you get an error saying that results were omitted because the metastore was slow, consider changing the latency to a higher number. | +| `primary-meta-store.mapped-databases` | No | List of databases to federate from the primary metastore; all other databases will be ignored. This property supports both full database names and [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) (both being case-insensitive). By default, all databases from the metastore are federated. | +| `primary-meta-store.mapped-tables` | No | List of mappings from databases to tables to federate from the primary metastore, similar to `mapped-databases`. By default, all tables are available. See `mapped-tables` configuration below. | +| `primary-meta-stores.hive-metastore-filter-hook` | No | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. | +| `primary-meta-stores.database-name-mapping` | No | BiDirectional Map of database names and mapped name, where key=`` and value=``. See the [Database Name Mapping](#database-name-mapping) section. | +| `primary-meta-stores.configuration-properties` | No | Map of the primary metastore personalized properties that will be added to the HiveConf used when creating the Thrift clients (they will be effect only on this client),the priority is higher than the properites of the same name in waggle-dance-server.yml. | +| `federated-meta-stores` | No | Possible empty list of read only federated metastores. | +| `federated-meta-stores[n].remote-meta-store-uris` | Yes | Thrift URIs of the federated read-only metastore. | +| `federated-meta-stores[n].name` | Yes | Name that uniquely identifies this metastore. Used internally. Cannot be empty. | +| `federated-meta-stores[n].database-prefix` | No | Prefix used to access this particular metastore and differentiate databases in it from databases in another metastore. Typically used if databases have the same name across metastores but federated access to them is still needed. The default prefix (i.e. if this value isn't explicitly set) is {federated-meta-stores[n].name} lowercased and postfixed with an underscore. For example if the metastore name was configured as "waggle" and no database prefix was provided but `PREFIXED` database resolution was used then the value of `database-prefix` would be "waggle_". | +| `federated-meta-stores[n].metastore-tunnel` | No | See metastore tunnel configuration values below. | +| `federated-meta-stores[n].latency` | No | Indicates the acceptable slowness of the metastore in **milliseconds** for increasing the default connection timeout. Default latency is `0` and should be changed if the metastore is particularly slow. If you get an error saying that results were omitted because the metastore was slow, consider changing the latency to a higher number. | +| `federated-meta-stores[n].mapped-databases` | No | List of databases to federate from this federated metastore, all other databases will be ignored. This property supports both full database names and [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) (both being case-insensitive). By default, all databases from the metastore are federated. | +| `federated-meta-stores[n].mapped-tables` | No | List of mappings from databases to tables to federate from this federated metastore, similar to `mapped-databases`. By default, all tables are available. See `mapped-tables` configuration below. | +| `federated-meta-stores[n].hive-metastore-filter-hook` | No | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. | +| `federated-meta-stores[n].database-name-mapping` | No | BiDirectional Map of database names and mapped names where key=`` and value=``. See the [Database Name Mapping](#database-name-mapping) section. | +| `federated-meta-stores[n].writable-database-white-list` | No | White-list of databases used to verify write access used in conjunction with `federated-meta-stores[n].access-control-type`. The list of databases should be listed without a `federated-meta-stores[n].database-prefix`. This property supports both full database names and (case-insensitive) [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html). | +| `federated-meta-stores[n].configuration-properties` | No | Map of the federate metastore personalized properties that will be added to the HiveConf used when creating the Thrift clients (they will be effect only on this client),the priority is higher than the properites of the same name in waggle-dance-server.yml. | #### Metastore tunnel The table below describes the metastore tunnel configuration values: diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index 600917f15..f825dd148 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -202,6 +202,15 @@ public HashBiMap getDatabaseNameBiMapping() { return databaseNameBiMapping; } + public Map getConfigurationProperties() { + return configurationProperties; + } + + public void setConfigurationProperties( + Map configurationProperties) { + this.configurationProperties = configurationProperties; + } + @Transient public MetaStoreStatus getStatus() { return status; @@ -243,13 +252,4 @@ public String toString() { .add("status", status) .toString(); } - - public Map getConfigurationProperties() { - return configurationProperties; - } - - public void setConfigurationProperties( - Map configurationProperties) { - this.configurationProperties = configurationProperties; - } } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index 90e907169..cd60bfb44 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -47,10 +47,10 @@ public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore if (waggleDanceConfiguration.getConfigurationProperties() != null) { properties.putAll(waggleDanceConfiguration.getConfigurationProperties()); } - //override per metastore - if(metaStore.getConfigurationProperties() != null) { + if (metaStore.getConfigurationProperties() != null) { properties.putAll(metaStore.getConfigurationProperties()); } + return newHiveInstance(metaStore, properties); } diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java index 9ef1d1f15..73ca5d968 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java @@ -71,7 +71,7 @@ public void setUp() { public void defaultFactory() { ArgumentCaptor hiveConfCaptor = ArgumentCaptor.forClass(HiveConf.class); FederatedMetaStore fed1 = newFederatedInstance("fed1", THRIFT_URI); - fed1.setConfigurationProperties(Collections.singletonMap(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname,"hive/_HOST@HADOOP.COM")); + fed1.setConfigurationProperties(Collections.singletonMap(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, "hive/_HOST@HADOOP.COM")); factory.newInstance(fed1); verify(defaultMetaStoreClientFactory).newInstance(hiveConfCaptor.capture(), eq( "waggledance-fed1"), eq(3), eq(2000)); diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java index fa1c88961..1882de258 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java @@ -212,7 +212,7 @@ public void saveFederationWriteFederations() throws Exception { MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2")); newFederatedInstance.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2)); newFederatedInstance.setHiveMetastoreFilterHook("filter.hook.class"); - newFederatedInstance.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal","hive/_HOST@REALM")); + newFederatedInstance.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM")); storage.insert(newFederatedInstance); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); @@ -296,7 +296,7 @@ public void savePrimaryWriteFederations() throws Exception { MappedTables mappedTables1 = new MappedTables("db1", Lists.newArrayList("tbl1")); MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2")); primaryMetaStore.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2)); - primaryMetaStore.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal","hive/_HOST@REALM")); + primaryMetaStore.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM")); storage.insert(primaryMetaStore); storage.insert(newFederatedInstance("hcom_2", "thrift://localhost:29083")); storage.saveFederation(); From f837b2eecdd179f8b605250167a02147d1bfb9c6 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Sat, 25 May 2024 11:35:44 +0800 Subject: [PATCH 09/13] Revert --- .../hotels/bdp/waggledance/api/model/AbstractMetaStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index f825dd148..010433a5c 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -58,8 +58,8 @@ public abstract class AbstractMetaStore { private @NotNull AccessControlType accessControlType = AccessControlType.READ_ONLY; private transient @JsonProperty @NotNull MetaStoreStatus status = MetaStoreStatus.UNKNOWN; private long latency = 0; - private transient @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); - private Map configurationProperties;// = Collections.emptyMap(); + private @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); + private Map configurationProperties = Collections.emptyMap(); public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) { this.name = name; From bc64c92deb8379e299a7ca8c23514542ced01bcb Mon Sep 17 00:00:00 2001 From: Patrick Duin Date: Mon, 27 May 2024 09:48:15 +0200 Subject: [PATCH 10/13] Update FederatedMetaStoreTest.java --- .../bdp/waggledance/api/model/FederatedMetaStoreTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java index 0f4fdc8ac..dcbe47103 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java @@ -72,7 +72,7 @@ public void nullDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":null,\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); From 8cb4b228b1426e12bfe1edf99f2a2b600925f2a8 Mon Sep 17 00:00:00 2001 From: Patrick Duin Date: Mon, 27 May 2024 09:48:35 +0200 Subject: [PATCH 11/13] Update PrimaryMetaStoreTest.java --- .../hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java index 4a3208fc3..1e1b1dc67 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java @@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":null,\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); From 5bd959ca3c8e781f6e04520de8303bc1ecb0c2c9 Mon Sep 17 00:00:00 2001 From: Patrick Duin Date: Mon, 27 May 2024 11:37:17 +0200 Subject: [PATCH 12/13] Update AbstractMetaStore.java using new HashMap so the generated Yaml doesn't generate an anchor (reference &id001) --- .../hotels/bdp/waggledance/api/model/AbstractMetaStore.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index 010433a5c..c78b63342 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -51,15 +51,15 @@ public abstract class AbstractMetaStore { private List writableDatabaseWhitelist; private List mappedDatabases; private @Valid List mappedTables; - private transient Map databaseNameMapping = Collections.emptyMap(); + private Map databaseNameMapping = new HashMap<>(); private @NotBlank String name; private @NotBlank String remoteMetaStoreUris; private @Valid MetastoreTunnel metastoreTunnel; private @NotNull AccessControlType accessControlType = AccessControlType.READ_ONLY; private transient @JsonProperty @NotNull MetaStoreStatus status = MetaStoreStatus.UNKNOWN; private long latency = 0; - private @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); - private Map configurationProperties = Collections.emptyMap(); + private transient @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); + private Map configurationProperties = new HashMap<>(); public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) { this.name = name; From 23258d2befa2a18233549c99bed22b6d7b609360 Mon Sep 17 00:00:00 2001 From: Patrick Duin Date: Mon, 27 May 2024 11:39:31 +0200 Subject: [PATCH 13/13] Update YamlFederatedMetaStoreStorageTest.java fixing test --- .../YamlFederatedMetaStoreStorageTest.java | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java index 1882de258..a39434a8e 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java @@ -216,7 +216,7 @@ public void saveFederationWriteFederations() throws Exception { storage.insert(newFederatedInstance); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(26)); + assertThat(lines.size(), is(25)); assertThat(lines.get(0), is("primary-meta-store:")); assertThat(lines.get(1), is(" access-control-type: READ_ONLY")); assertThat(lines.get(2), is(" database-prefix: ''")); @@ -227,22 +227,21 @@ public void saveFederationWriteFederations() throws Exception { assertThat(lines.get(7), is("- access-control-type: READ_ONLY")); assertThat(lines.get(8), is(" configuration-properties:")); assertThat(lines.get(9), is(" hive.metastore.kerberos.principal: hive/_HOST@REALM")); - assertThat(lines.get(10), is(" database-name-mapping: {}")); - assertThat(lines.get(11), is(" database-prefix: hcom_2_")); - assertThat(lines.get(12), is(" hive-metastore-filter-hook: filter.hook.class")); - assertThat(lines.get(13), is(" latency: 0")); - assertThat(lines.get(14), is(" mapped-databases:")); - assertThat(lines.get(15), is(" - db1")); - assertThat(lines.get(16), is(" - db2")); - assertThat(lines.get(17), is(" mapped-tables:")); - assertThat(lines.get(18), is(" - database: db1")); - assertThat(lines.get(19), is(" mapped-tables:")); - assertThat(lines.get(20), is(" - tbl1")); - assertThat(lines.get(21), is(" - database: db2")); - assertThat(lines.get(22), is(" mapped-tables:")); - assertThat(lines.get(23), is(" - tbl2")); - assertThat(lines.get(24), is(" name: hcom_2")); - assertThat(lines.get(25), is(" remote-meta-store-uris: thrift://localhost:29083")); + assertThat(lines.get(10), is(" database-prefix: hcom_2_")); + assertThat(lines.get(11), is(" hive-metastore-filter-hook: filter.hook.class")); + assertThat(lines.get(12), is(" latency: 0")); + assertThat(lines.get(13), is(" mapped-databases:")); + assertThat(lines.get(14), is(" - db1")); + assertThat(lines.get(15), is(" - db2")); + assertThat(lines.get(16), is(" mapped-tables:")); + assertThat(lines.get(17), is(" - database: db1")); + assertThat(lines.get(18), is(" mapped-tables:")); + assertThat(lines.get(19), is(" - tbl1")); + assertThat(lines.get(20), is(" - database: db2")); + assertThat(lines.get(21), is(" mapped-tables:")); + assertThat(lines.get(22), is(" - tbl2")); + assertThat(lines.get(23), is(" name: hcom_2")); + assertThat(lines.get(24), is(" remote-meta-store-uris: thrift://localhost:29083")); } @Test @@ -301,7 +300,7 @@ public void savePrimaryWriteFederations() throws Exception { storage.insert(newFederatedInstance("hcom_2", "thrift://localhost:29083")); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(25)); + assertThat(lines.size(), is(24)); assertThat(lines.get(0), is("primary-meta-store:")); assertThat(lines.get(1), is(" access-control-type: READ_ONLY")); assertThat(lines.get(2), is(" configuration-properties:")); @@ -322,11 +321,10 @@ public void savePrimaryWriteFederations() throws Exception { assertThat(lines.get(17), is(" remote-meta-store-uris: thrift://localhost:19083")); assertThat(lines.get(18), is("federated-meta-stores:")); assertThat(lines.get(19), is("- access-control-type: READ_ONLY")); - assertThat(lines.get(20), is(" database-name-mapping: {}")); - assertThat(lines.get(21), is(" database-prefix: hcom_2_")); - assertThat(lines.get(22), is(" latency: 0")); - assertThat(lines.get(23), is(" name: hcom_2")); - assertThat(lines.get(24), is(" remote-meta-store-uris: thrift://localhost:29083")); + assertThat(lines.get(20), is(" database-prefix: hcom_2_")); + assertThat(lines.get(21), is(" latency: 0")); + assertThat(lines.get(22), is(" name: hcom_2")); + assertThat(lines.get(23), is(" remote-meta-store-uris: thrift://localhost:29083")); } private PrimaryMetaStore newPrimaryInstance(String name, String remoteMetaStoreUris) {