Add personalized configuration parameters for each metastore.

waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java

@@ -51,14 +51,15 @@ public abstract class AbstractMetaStore {
   private List<String> writableDatabaseWhitelist;
   private List<String> mappedDatabases;
   private @Valid List<MappedTables> mappedTables;
-  private Map<String, String> databaseNameMapping = Collections.emptyMap();
+  private transient Map<String, String> databaseNameMapping = Collections.emptyMap();
   private @NotBlank String name;
   private @NotBlank String remoteMetaStoreUris;
   private @Valid MetastoreTunnel metastoreTunnel;
   private @NotNull AccessControlType accessControlType = AccessControlType.READ_ONLY;
   private transient @JsonProperty @NotNull MetaStoreStatus status = MetaStoreStatus.UNKNOWN;
   private long latency = 0;
   private transient @JsonIgnore HashBiMap<String, String> databaseNameBiMapping = HashBiMap.create();
+  private Map<String, String> configurationProperties;// = Collections.emptyMap();
 
   public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) {
     this.name = name;
@@ -201,6 +202,15 @@ public HashBiMap<String, String> getDatabaseNameBiMapping() {
     return databaseNameBiMapping;
   }
 
+  public Map<String, String> getConfigurationProperties() {
+    return configurationProperties;
+  }
+
+  public void setConfigurationProperties(
+          Map<String, String> configurationProperties) {
+    this.configurationProperties = configurationProperties;
+  }
+
   @Transient
   public MetaStoreStatus getStatus() {
     return status;
@@ -242,5 +252,4 @@ public String toString() {
         .add("status", status)
         .toString();
   }
-
 }

waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java

@@ -72,7 +72,7 @@ public void nullDatabasePrefix() {
 
   @Test
   public void toJson() throws Exception {
-    String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
+    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":null,\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
     ObjectMapper mapper = new ObjectMapper();
     // Sorting to get deterministic test behaviour
     mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);

waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java

@@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() {
 
   @Test
   public void toJson() throws Exception {
-    String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
+    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":null,\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
     ObjectMapper mapper = new ObjectMapper();
     // Sorting to get deterministic test behaviour
     mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java

@@ -47,6 +47,10 @@ public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore
     if (waggleDanceConfiguration.getConfigurationProperties() != null) {
       properties.putAll(waggleDanceConfiguration.getConfigurationProperties());
     }
+    if (metaStore.getConfigurationProperties() != null) {
+      properties.putAll(metaStore.getConfigurationProperties());
+    }
+
     return newHiveInstance(metaStore, properties);
   }
 

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java

@@ -24,10 +24,12 @@
 
 import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import com.hotels.bdp.waggledance.api.model.FederatedMetaStore;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.junit.Before;
@@ -68,8 +70,9 @@ public void setUp() {
   @Test
   public void defaultFactory() {
     ArgumentCaptor<HiveConf> hiveConfCaptor = ArgumentCaptor.forClass(HiveConf.class);
-
-    factory.newInstance(newFederatedInstance("fed1", THRIFT_URI));
+    FederatedMetaStore fed1 = newFederatedInstance("fed1", THRIFT_URI);
+    fed1.setConfigurationProperties(Collections.singletonMap(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, "hive/_HOST@HADOOP.COM"));
+    factory.newInstance(fed1);
     verify(defaultMetaStoreClientFactory).newInstance(hiveConfCaptor.capture(), eq(
         "waggledance-fed1"), eq(3), eq(2000));
     verifyNoInteractions(tunnelingMetaStoreClientFactory);
@@ -80,6 +83,7 @@ public void defaultFactory() {
     assertThat(hiveConf.getTimeVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS), is(5L));
     assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT), is(true));
     assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_COMPACT_PROTOCOL), is(false));
+    assertThat(hiveConf.getVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL), is("hive/_HOST@HADOOP.COM"));
   }
 
   @Test

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java

@@ -23,6 +23,7 @@
 import java.io.File;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.util.Collections;
 import java.util.List;
 
 import javax.validation.ConstraintViolationException;
@@ -211,10 +212,11 @@ public void saveFederationWriteFederations() throws Exception {
     MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2"));
     newFederatedInstance.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2));
     newFederatedInstance.setHiveMetastoreFilterHook("filter.hook.class");
+    newFederatedInstance.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM"));
     storage.insert(newFederatedInstance);
     storage.saveFederation();
     List<String> lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8);
-    assertThat(lines.size(), is(24));
+    assertThat(lines.size(), is(26));
     assertThat(lines.get(0), is("primary-meta-store:"));
     assertThat(lines.get(1), is("  access-control-type: READ_ONLY"));
     assertThat(lines.get(2), is("  database-prefix: ''"));
@@ -223,22 +225,24 @@ public void saveFederationWriteFederations() throws Exception {
     assertThat(lines.get(5), is("  remote-meta-store-uris: thrift://localhost:19083"));
     assertThat(lines.get(6), is("federated-meta-stores:"));
     assertThat(lines.get(7), is("- access-control-type: READ_ONLY"));
-    assertThat(lines.get(8), is("  database-name-mapping: {}"));
-    assertThat(lines.get(9), is("  database-prefix: hcom_2_"));
-    assertThat(lines.get(10), is("  hive-metastore-filter-hook: filter.hook.class"));
-    assertThat(lines.get(11), is("  latency: 0"));
-    assertThat(lines.get(12), is("  mapped-databases:"));
-    assertThat(lines.get(13), is("  - db1"));
-    assertThat(lines.get(14), is("  - db2"));
-    assertThat(lines.get(15), is("  mapped-tables:"));
-    assertThat(lines.get(16), is("  - database: db1"));
-    assertThat(lines.get(17), is("    mapped-tables:"));
-    assertThat(lines.get(18), is("    - tbl1"));
-    assertThat(lines.get(19), is("  - database: db2"));
-    assertThat(lines.get(20), is("    mapped-tables:"));
-    assertThat(lines.get(21), is("    - tbl2"));
-    assertThat(lines.get(22), is("  name: hcom_2"));
-    assertThat(lines.get(23), is("  remote-meta-store-uris: thrift://localhost:29083"));
+    assertThat(lines.get(8), is("  configuration-properties:"));
+    assertThat(lines.get(9), is("    hive.metastore.kerberos.principal: hive/_HOST@REALM"));
+    assertThat(lines.get(10), is("  database-name-mapping: {}"));
+    assertThat(lines.get(11), is("  database-prefix: hcom_2_"));
+    assertThat(lines.get(12), is("  hive-metastore-filter-hook: filter.hook.class"));
+    assertThat(lines.get(13), is("  latency: 0"));
+    assertThat(lines.get(14), is("  mapped-databases:"));
+    assertThat(lines.get(15), is("  - db1"));
+    assertThat(lines.get(16), is("  - db2"));
+    assertThat(lines.get(17), is("  mapped-tables:"));
+    assertThat(lines.get(18), is("  - database: db1"));
+    assertThat(lines.get(19), is("    mapped-tables:"));
+    assertThat(lines.get(20), is("    - tbl1"));
+    assertThat(lines.get(21), is("  - database: db2"));
+    assertThat(lines.get(22), is("    mapped-tables:"));
+    assertThat(lines.get(23), is("    - tbl2"));
+    assertThat(lines.get(24), is("  name: hcom_2"));
+    assertThat(lines.get(25), is("  remote-meta-store-uris: thrift://localhost:29083"));
   }
 
   @Test
@@ -292,34 +296,37 @@ public void savePrimaryWriteFederations() throws Exception {
     MappedTables mappedTables1 = new MappedTables("db1", Lists.newArrayList("tbl1"));
     MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2"));
     primaryMetaStore.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2));
+    primaryMetaStore.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM"));
     storage.insert(primaryMetaStore);
     storage.insert(newFederatedInstance("hcom_2", "thrift://localhost:29083"));
     storage.saveFederation();
     List<String> lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8);
-    assertThat(lines.size(), is(23));
+    assertThat(lines.size(), is(25));
     assertThat(lines.get(0), is("primary-meta-store:"));
     assertThat(lines.get(1), is("  access-control-type: READ_ONLY"));
-    assertThat(lines.get(2), is("  database-prefix: ''"));
-    assertThat(lines.get(3), is("  latency: 0"));
-    assertThat(lines.get(4), is("  mapped-databases:"));
-    assertThat(lines.get(5), is("  - db1"));
-    assertThat(lines.get(6), is("  - db2"));
-    assertThat(lines.get(7), is("  mapped-tables:"));
-    assertThat(lines.get(8), is("  - database: db1"));
-    assertThat(lines.get(9), is("    mapped-tables:"));
-    assertThat(lines.get(10), is("    - tbl1"));
-    assertThat(lines.get(11), is("  - database: db2"));
-    assertThat(lines.get(12), is("    mapped-tables:"));
-    assertThat(lines.get(13), is("    - tbl2"));
-    assertThat(lines.get(14), is("  name: hcom_1"));
-    assertThat(lines.get(15), is("  remote-meta-store-uris: thrift://localhost:19083"));
-    assertThat(lines.get(16), is("federated-meta-stores:"));
-    assertThat(lines.get(17), is("- access-control-type: READ_ONLY"));
-    assertThat(lines.get(18), is("  database-name-mapping: {}"));
-    assertThat(lines.get(19), is("  database-prefix: hcom_2_"));
-    assertThat(lines.get(20), is("  latency: 0"));
-    assertThat(lines.get(21), is("  name: hcom_2"));
-    assertThat(lines.get(22), is("  remote-meta-store-uris: thrift://localhost:29083"));
+    assertThat(lines.get(2), is("  configuration-properties:"));
+    assertThat(lines.get(3), is("    hive.metastore.kerberos.principal: hive/_HOST@REALM"));
+    assertThat(lines.get(4), is("  database-prefix: ''"));
+    assertThat(lines.get(5), is("  latency: 0"));
+    assertThat(lines.get(6), is("  mapped-databases:"));
+    assertThat(lines.get(7), is("  - db1"));
+    assertThat(lines.get(8), is("  - db2"));
+    assertThat(lines.get(9), is("  mapped-tables:"));
+    assertThat(lines.get(10), is("  - database: db1"));
+    assertThat(lines.get(11), is("    mapped-tables:"));
+    assertThat(lines.get(12), is("    - tbl1"));
+    assertThat(lines.get(13), is("  - database: db2"));
+    assertThat(lines.get(14), is("    mapped-tables:"));
+    assertThat(lines.get(15), is("    - tbl2"));
+    assertThat(lines.get(16), is("  name: hcom_1"));
+    assertThat(lines.get(17), is("  remote-meta-store-uris: thrift://localhost:19083"));
+    assertThat(lines.get(18), is("federated-meta-stores:"));
+    assertThat(lines.get(19), is("- access-control-type: READ_ONLY"));
+    assertThat(lines.get(20), is("  database-name-mapping: {}"));
+    assertThat(lines.get(21), is("  database-prefix: hcom_2_"));
+    assertThat(lines.get(22), is("  latency: 0"));
+    assertThat(lines.get(23), is("  name: hcom_2"));
+    assertThat(lines.get(24), is("  remote-meta-store-uris: thrift://localhost:29083"));
   }
 
   private PrimaryMetaStore newPrimaryInstance(String name, String remoteMetaStoreUris) {