From 19f57a5892e8be719f5d4e421645292241fdb2a5 Mon Sep 17 00:00:00 2001 From: Patrick Duin Date: Thu, 8 Feb 2024 21:45:32 +0100 Subject: [PATCH] Added extra option to add readOnly thrift HMS uri (#308) * Added extra option to add readOnly thrift HMS uri which will be called on read only calls for better spread of traffic --- CHANGELOG.md | 4 + README.md | 1 + pom.xml | 2 +- waggle-dance-api/pom.xml | 2 +- .../api/model/AbstractMetaStore.java | 13 +- .../api/model/FederatedMetaStoreTest.java | 4 +- .../api/model/PrimaryMetaStoreTest.java | 4 +- waggle-dance-boot/pom.xml | 2 +- waggle-dance-core/pom.xml | 2 +- ...ThriftHiveMetastoreIfaceClientFactory.java | 23 ++- .../SplitTrafficMetastoreClientFactory.java | 95 +++++++++++++ .../client/adapter/MetastoreIfaceAdapter.java | 2 +- ...atibleThriftHiveMetastoreIfaceFactory.java | 2 +- .../bdp/waggledance/context/CommonBeans.java | 12 +- .../server/FederatedHMSHandler.java | 2 +- ...ftHiveMetastoreIfaceClientFactoryTest.java | 29 +++- ...plitTrafficMetastoreClientFactoryTest.java | 134 ++++++++++++++++++ .../MetaStoreMappingFactoryImplTest.java | 5 +- waggle-dance-integration-tests/pom.xml | 2 +- waggle-dance-rest/pom.xml | 2 +- waggle-dance-rpm/pom.xml | 2 +- waggle-dance/pom.xml | 2 +- 22 files changed, 318 insertions(+), 28 deletions(-) create mode 100644 waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactory.java create mode 100644 waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactoryTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 867c17d6f..a49388ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ +## [3.12.0] - 2023-02-08 +### Added +- Added optional `primary-meta-store.read-only-remote-meta-store-uris` config to allow traffic to be diverted based on calls made. See README.md. + ## [3.11.7] - 2023-11-30 ### Changed - Fixed log statement that was not logging the exception correctly. diff --git a/README.md b/README.md index 6b3663c83..39059e863 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ The table below describes all the available configuration values for Waggle Danc | `primary-meta-store.hive-metastore-filter-hook` | No | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. | | `primary-meta-store.database-name-mapping` | No | BiDirectional Map of database names and mapped name, where key=`` and value=``. See the [Database Name Mapping](#database-name-mapping) section.| | `primary-meta-store.glue-config` | No | Can be used instead of `remote-meta-store-uris` to federate to an AWS Glue Catalog ([AWS Glue](https://docs.aws.amazon.com/glue/index.html). See the [Federate to AWS Glue Catalog](#federate-to-aws-glue-catalog) section.| +| `primary-meta-store.read-only-remote-meta-store-uris` | No | Can be used to configure an extra read-only endpoint for the primary Metastore. This is an optimization if your environment runs separate Metastore endpoints and traffic needs to be diverted efficiently. Waggle Dance will direct traffic to the read-write or read-only endpoints based on the call being done. For instance `get_table` will be a read-only call but `alter_table` will be forwarded to the read-write Metastore.| | `federated-meta-stores` | No | Possible empty list of read only federated metastores. | | `federated-meta-stores[n].remote-meta-store-uris` | Yes | Thrift URIs of the federated read-only metastore. | | `federated-meta-stores[n].name` | Yes | Name that uniquely identifies this metastore. Used internally. Cannot be empty. | diff --git a/pom.xml b/pom.xml index b93efa34e..875c50d43 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT pom Waggle Dance Parent Hive Metastore federation service. diff --git a/waggle-dance-api/pom.xml b/waggle-dance-api/pom.xml index 41542bdaf..b085cd90f 100644 --- a/waggle-dance-api/pom.xml +++ b/waggle-dance-api/pom.xml @@ -4,7 +4,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT waggle-dance-api diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index e7eec1616..c225c4358 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2023 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,6 +57,7 @@ public abstract class AbstractMetaStore { private long latency = 0; private transient @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); private GlueConfig glueConfig; + private String readOnlyRemoteMetaStoreUris; public AbstractMetaStore() {} @@ -124,6 +125,14 @@ public void setRemoteMetaStoreUris(String remoteMetaStoreUris) { this.remoteMetaStoreUris = remoteMetaStoreUris; } + public String getReadOnlyRemoteMetaStoreUris() { + return readOnlyRemoteMetaStoreUris; + } + + public void setReadOnlyRemoteMetaStoreUris(String readOnlyRemoteMetaStoreUris) { + this.readOnlyRemoteMetaStoreUris = readOnlyRemoteMetaStoreUris; + } + public MetastoreTunnel getMetastoreTunnel() { return metastoreTunnel; } @@ -244,6 +253,7 @@ public String toString() { .add("databasePrefix", databasePrefix) .add("federationType", getFederationType()) .add("remoteMetaStoreUris", remoteMetaStoreUris) + .add("readOnlyRemoteMetaStoreUris", readOnlyRemoteMetaStoreUris) .add("metastoreTunnel", metastoreTunnel) .add("accessControlType", accessControlType) .add("writableDatabaseWhiteList", writableDatabaseWhitelist) @@ -252,4 +262,5 @@ public String toString() { .toString(); } + } diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java index 2dea5f2a7..4df11f6eb 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2022 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,7 +72,7 @@ public void nullDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"glueConfig\":null,\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"glueConfig\":null,\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"readOnlyRemoteMetaStoreUris\":null,\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java index 1b44d7718..212787d45 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2022 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"glueConfig\":null,\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"glueConfig\":null,\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"readOnlyRemoteMetaStoreUris\":null,\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-boot/pom.xml b/waggle-dance-boot/pom.xml index f51958eeb..8b5abed85 100644 --- a/waggle-dance-boot/pom.xml +++ b/waggle-dance-boot/pom.xml @@ -4,7 +4,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT waggle-dance-boot diff --git a/waggle-dance-core/pom.xml b/waggle-dance-core/pom.xml index f46ee4210..f5892a28e 100644 --- a/waggle-dance-core/pom.xml +++ b/waggle-dance-core/pom.xml @@ -4,7 +4,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT waggle-dance-core diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index d6b06e433..5a3b2c3b2 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2022 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,16 +44,19 @@ public class CloseableThriftHiveMetastoreIfaceClientFactory { private final int defaultConnectionTimeout = (int) TimeUnit.SECONDS.toMillis(2L); private final WaggleDanceConfiguration waggleDanceConfiguration; private final GlueClientFactory glueClientFactory; + private final SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory; public CloseableThriftHiveMetastoreIfaceClientFactory( TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory, DefaultMetaStoreClientFactory defaultMetaStoreClientFactory, GlueClientFactory glueClientFactory, - WaggleDanceConfiguration waggleDanceConfiguration) { + WaggleDanceConfiguration waggleDanceConfiguration, + SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory) { this.tunnelingMetaStoreClientFactory = tunnelingMetaStoreClientFactory; this.defaultMetaStoreClientFactory = defaultMetaStoreClientFactory; this.glueClientFactory = glueClientFactory; this.waggleDanceConfiguration = waggleDanceConfiguration; + this.splitTrafficMetaStoreClientFactory = splitTrafficMetaStoreClientFactory; } public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore) { @@ -64,14 +67,24 @@ public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore if (metaStore.getGlueConfig() != null) { return newGlueInstance(metaStore, properties); } - return newHiveInstance(metaStore, properties); + String name = metaStore.getName().toLowerCase(Locale.ROOT); + if (metaStore.getReadOnlyRemoteMetaStoreUris() != null) { + CloseableThriftHiveMetastoreIface readWrite = newHiveInstance(metaStore, name, metaStore.getRemoteMetaStoreUris(), + properties); + CloseableThriftHiveMetastoreIface readOnly = newHiveInstance(metaStore, name + "_ro", + metaStore.getReadOnlyRemoteMetaStoreUris(), properties); + return splitTrafficMetaStoreClientFactory.newInstance(readWrite, readOnly); + + } + return newHiveInstance(metaStore, name, metaStore.getRemoteMetaStoreUris(), properties); } private CloseableThriftHiveMetastoreIface newHiveInstance( AbstractMetaStore metaStore, + String name, + String metaStoreUris, Map properties) { - String uris = MetaStoreUriNormaliser.normaliseMetaStoreUris(metaStore.getRemoteMetaStoreUris()); - String name = metaStore.getName().toLowerCase(Locale.ROOT); + String uris = MetaStoreUriNormaliser.normaliseMetaStoreUris(metaStoreUris); // Connection timeout should not be less than 1 // A timeout of zero is interpreted as an infinite timeout, so this is avoided int connectionTimeout = Math.max(1, defaultConnectionTimeout + (int) metaStore.getLatency()); diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactory.java new file mode 100644 index 000000000..ec86d483c --- /dev/null +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactory.java @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2016-2024 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.hotels.bdp.waggledance.client; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class splits the traffic for read only calls (get* for instance getTable, getPartition) to the readOnly client + * and everything else will go to readWrite client. + */ +public class SplitTrafficMetastoreClientFactory { + + static final Class[] INTERFACES = new Class[] { CloseableThriftHiveMetastoreIface.class }; + + private static class SplitTrafficClientInvocationHandler implements InvocationHandler { + + private static Logger log = LoggerFactory + .getLogger(SplitTrafficMetastoreClientFactory.SplitTrafficClientInvocationHandler.class); + + private final CloseableThriftHiveMetastoreIface readWrite; + private final CloseableThriftHiveMetastoreIface readOnly; + + public SplitTrafficClientInvocationHandler( + CloseableThriftHiveMetastoreIface readWrite, + CloseableThriftHiveMetastoreIface readOnly) { + this.readWrite = readWrite; + this.readOnly = readOnly; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + switch (method.getName()) { + case "isOpen": + return readWrite.isOpen() && readOnly.isOpen(); + case "close": + try { + readWrite.close(); + } finally { + readOnly.close(); + } + return null; + case "set_ugi": + Object result = doRealCall(readWrite, method, args); + // we skip the result for readOnly (it should always be the same). + doRealCall(readOnly, method, args); + return result; + default: + if (method.getName().startsWith("get")) { + log.info("Calling {}.{}", "readOnly", method.getName()); + return doRealCall(readOnly, method, args); + } + log.info("Calling {}.{}", "readWrite", method.getName()); + return doRealCall(readWrite, method, args); + } + } + + private Object doRealCall(CloseableThriftHiveMetastoreIface client, Method method, Object[] args) + throws IllegalAccessException, Throwable { + try { + return method.invoke(client, args); + } catch (InvocationTargetException e) { + Throwable realException = e.getTargetException(); + throw realException; + } + } + } + + public CloseableThriftHiveMetastoreIface newInstance( + CloseableThriftHiveMetastoreIface readWrite, + CloseableThriftHiveMetastoreIface readOnly) { + return (CloseableThriftHiveMetastoreIface) Proxy + .newProxyInstance(getClass().getClassLoader(), INTERFACES, + new SplitTrafficClientInvocationHandler(readWrite, readOnly)); + } + +} diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/adapter/MetastoreIfaceAdapter.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/adapter/MetastoreIfaceAdapter.java index 23d238309..8317539f4 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/adapter/MetastoreIfaceAdapter.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/adapter/MetastoreIfaceAdapter.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2022 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/compatibility/HiveCompatibleThriftHiveMetastoreIfaceFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/compatibility/HiveCompatibleThriftHiveMetastoreIfaceFactory.java index 3ce8ee3f5..1d9bc3b75 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/compatibility/HiveCompatibleThriftHiveMetastoreIfaceFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/compatibility/HiveCompatibleThriftHiveMetastoreIfaceFactory.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2019 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java index 48db6e1e5..f92efd167 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2023 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIfaceClientFactory; import com.hotels.bdp.waggledance.client.DefaultMetaStoreClientFactory; import com.hotels.bdp.waggledance.client.GlueClientFactory; +import com.hotels.bdp.waggledance.client.SplitTrafficMetastoreClientFactory; import com.hotels.bdp.waggledance.client.tunnelling.TunnelingMetaStoreClientFactory; import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration; import com.hotels.bdp.waggledance.mapping.model.ASTQueryMapping; @@ -53,11 +54,16 @@ public PrefixNamingStrategy prefixNamingStrategy(WaggleDanceConfiguration waggle return new LowerCasePrefixNamingStrategy(); } + @Bean + public SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory() { + return new SplitTrafficMetastoreClientFactory(); + } + @Bean public CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory( - WaggleDanceConfiguration waggleDanceConfiguration) { + WaggleDanceConfiguration waggleDanceConfiguration, SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory) { return new CloseableThriftHiveMetastoreIfaceClientFactory(new TunnelingMetaStoreClientFactory(), - new DefaultMetaStoreClientFactory(), new GlueClientFactory(), waggleDanceConfiguration); + new DefaultMetaStoreClientFactory(), new GlueClientFactory(), waggleDanceConfiguration, splitTrafficMetaStoreClientFactory); } @Bean diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java index 9305fffd5..46acabacd 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2022 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java index 3e00712d2..06fa2279c 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2022 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,12 +18,15 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.isA; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; +import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newPrimaryInstance; import java.util.HashMap; import java.util.Map; @@ -43,6 +46,7 @@ import com.hotels.bdp.waggledance.api.model.AbstractMetaStore; import com.hotels.bdp.waggledance.api.model.FederatedMetaStore; import com.hotels.bdp.waggledance.api.model.GlueConfig; +import com.hotels.bdp.waggledance.api.model.PrimaryMetaStore; import com.hotels.bdp.waggledance.client.adapter.MetastoreIfaceAdapter; import com.hotels.bdp.waggledance.client.tunnelling.TunnelingMetaStoreClientFactory; import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration; @@ -52,6 +56,7 @@ public class CloseableThriftHiveMetastoreIfaceClientFactoryTest { private static final String THRIFT_URI = "thrift://host:port"; + private static final String THRIFT_URI_READ_ONLY = "thrift://host-read-only:port"; private CloseableThriftHiveMetastoreIfaceClientFactory factory; private @Mock TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory; @@ -60,6 +65,7 @@ public class CloseableThriftHiveMetastoreIfaceClientFactoryTest { private @Mock WaggleDanceConfiguration waggleDanceConfiguration; private final Map configurationProperties = new HashMap<>(); private @Mock AWSCatalogMetastoreClient glueClient; + private @Mock SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory; @Before public void setUp() { @@ -70,7 +76,7 @@ public void setUp() { configurationProperties.put(ConfVars.METASTORE_USE_THRIFT_COMPACT_PROTOCOL.varname, "false"); when(waggleDanceConfiguration.getConfigurationProperties()).thenReturn(configurationProperties); factory = new CloseableThriftHiveMetastoreIfaceClientFactory(tunnelingMetaStoreClientFactory, - defaultMetaStoreClientFactory, glueClientFactory, waggleDanceConfiguration); + defaultMetaStoreClientFactory, glueClientFactory, waggleDanceConfiguration, splitTrafficMetaStoreClientFactory); } @Test @@ -90,6 +96,25 @@ public void defaultFactory() { assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_COMPACT_PROTOCOL), is(false)); } + @Test + public void splitTrafficFactory() { + PrimaryMetaStore metaStore = newPrimaryInstance("hms", THRIFT_URI); + metaStore.setReadOnlyRemoteMetaStoreUris(THRIFT_URI_READ_ONLY); + CloseableThriftHiveMetastoreIface readWriteClient = mock(CloseableThriftHiveMetastoreIface.class); + //Using 'any(HiveConf.class); generic matcher because HiveConf doesn't implement equals. + when(defaultMetaStoreClientFactory + .newInstance(any(HiveConf.class), eq("waggledance-hms"), eq(3), eq(2000))).thenReturn(readWriteClient); + CloseableThriftHiveMetastoreIface readOnlyclient = mock(CloseableThriftHiveMetastoreIface.class); + when(defaultMetaStoreClientFactory + .newInstance(any(HiveConf.class), eq("waggledance-hms_ro"), eq(3), eq(2000))).thenReturn(readOnlyclient); + + factory.newInstance(metaStore); + + + verify(splitTrafficMetaStoreClientFactory).newInstance(readWriteClient, readOnlyclient); + verifyNoInteractions(tunnelingMetaStoreClientFactory); + } + @Test public void tunnelingFactory() { MetastoreTunnel metastoreTunnel = new MetastoreTunnel(); diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactoryTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactoryTest.java new file mode 100644 index 000000000..05b25faf2 --- /dev/null +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/SplitTrafficMetastoreClientFactoryTest.java @@ -0,0 +1,134 @@ +/** + * Copyright (C) 2016-2024 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.hotels.bdp.waggledance.client; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.Table; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class SplitTrafficMetastoreClientFactoryTest { + + private @Mock CloseableThriftHiveMetastoreIface readWrite; + private @Mock CloseableThriftHiveMetastoreIface readOnly; + private @Mock Table readTable; + private @Mock Table writeTable; + + @Test + public void new_instance_getTable() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + when(readOnly.get_table("a", "b")).thenReturn(readTable); + + Table table = client.get_table("a", "b"); + + assertThat(table, is(readTable)); + verifyNoInteractions(readWrite); + } + + @Test + public void new_instance_alterTable() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + + client.alter_table("a", "b", writeTable); + + verify(readWrite).alter_table("a", "b", writeTable); + verifyNoInteractions(readOnly); + } + + @Test + public void new_instance_close() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + + client.close(); + + verify(readWrite).close(); + verify(readOnly).close(); + } + + @Test + public void new_instance_set_ugi() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + + List expected = Arrays.asList("result"); + when(readOnly.set_ugi("a", Arrays.asList("b"))).thenReturn(expected); + when(readWrite.set_ugi("a", Arrays.asList("b"))).thenReturn(expected); + List result = client.set_ugi("a", Arrays.asList("b")); + + assertThat(result, is(expected)); + verify(readOnly).set_ugi("a", Arrays.asList("b")); + verify(readWrite).set_ugi("a", Arrays.asList("b")); + } + + @Test + public void new_instance_isOpen_true() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + when(readOnly.isOpen()).thenReturn(true); + when(readWrite.isOpen()).thenReturn(true); + boolean isOpen = client.isOpen(); + + assertThat(isOpen, is(true)); + } + + @Test + public void new_instance_isOpen_false() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + when(readWrite.isOpen()).thenReturn(false); + boolean isOpen = client.isOpen(); + + assertThat(isOpen, is(false)); + verifyNoInteractions(readOnly); + } + + @Test + public void new_instance_isOpen_readOnly_false() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + when(readOnly.isOpen()).thenReturn(false); + when(readWrite.isOpen()).thenReturn(true); + boolean isOpen = client.isOpen(); + + assertThat(isOpen, is(false)); + } + + @Test + public void new_instance_isOpen_readWrite_false() throws Exception { + CloseableThriftHiveMetastoreIface client = new SplitTrafficMetastoreClientFactory() + .newInstance(readWrite, readOnly); + when(readWrite.isOpen()).thenReturn(false); + boolean isOpen = client.isOpen(); + + assertThat(isOpen, is(false)); + verifyNoInteractions(readOnly); + } + +} diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index d11c37640..2764bc17b 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2022 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,6 +43,7 @@ import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIfaceClientFactory; import com.hotels.bdp.waggledance.client.DefaultMetaStoreClientFactory; import com.hotels.bdp.waggledance.client.GlueClientFactory; +import com.hotels.bdp.waggledance.client.SplitTrafficMetastoreClientFactory; import com.hotels.bdp.waggledance.client.tunnelling.TunnelingMetaStoreClientFactory; import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration; import com.hotels.bdp.waggledance.mapping.service.PrefixNamingStrategy; @@ -61,7 +62,7 @@ public class MetaStoreMappingFactoryImplTest { private @Mock AccessControlHandlerFactory accessControlHandlerFactory; private final CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory = new CloseableThriftHiveMetastoreIfaceClientFactory( new TunnelingMetaStoreClientFactory(), new DefaultMetaStoreClientFactory(), new GlueClientFactory(), - new WaggleDanceConfiguration()); + new WaggleDanceConfiguration(), new SplitTrafficMetastoreClientFactory()); private MetaStoreMappingFactoryImpl factory; diff --git a/waggle-dance-integration-tests/pom.xml b/waggle-dance-integration-tests/pom.xml index 0081ba902..9e6fb0625 100644 --- a/waggle-dance-integration-tests/pom.xml +++ b/waggle-dance-integration-tests/pom.xml @@ -4,7 +4,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT waggle-dance-integration-tests diff --git a/waggle-dance-rest/pom.xml b/waggle-dance-rest/pom.xml index 156556558..35f61dab4 100644 --- a/waggle-dance-rest/pom.xml +++ b/waggle-dance-rest/pom.xml @@ -4,7 +4,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT waggle-dance-rest diff --git a/waggle-dance-rpm/pom.xml b/waggle-dance-rpm/pom.xml index 74ec86224..36fbeb64e 100644 --- a/waggle-dance-rpm/pom.xml +++ b/waggle-dance-rpm/pom.xml @@ -4,7 +4,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT waggle-dance-rpm diff --git a/waggle-dance/pom.xml b/waggle-dance/pom.xml index 0095eb3dc..c33ac84bc 100644 --- a/waggle-dance/pom.xml +++ b/waggle-dance/pom.xml @@ -4,7 +4,7 @@ com.hotels waggle-dance-parent - 3.11.8-SNAPSHOT + 3.12.0-SNAPSHOT waggle-dance