Add hive conf cache to CloseableThriftHiveMetastoreIfaceClientFactory (…

…#327) * Add hive conf cache
ExpediaGroup · Jul 23, 2024 · 6c12b4a · 6c12b4a
1 parent 3cd93da
commit 6c12b4a
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,6 @@
+## [3.13.2] - 2024-07-23
+### Fix
+- Add HiveConf cache to `CloseableThriftHiveMetastoreIfaceClientFactory` to prevent threads block. See [#325](https://github.com/ExpediaGroup/waggle-dance/issues/325)
 
 ## [3.13.1] - 2024-05-06
 ### Fix

diff --git a/...ava/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/...ava/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java
@@ -21,8 +21,10 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 
@@ -45,6 +47,7 @@ public class CloseableThriftHiveMetastoreIfaceClientFactory implements ThriftCli
   private final WaggleDanceConfiguration waggleDanceConfiguration;
   private final GlueClientFactory glueClientFactory;
   private final SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory;
+  private final ConcurrentHashMap<String, HiveConf> cachedHiveConf = new ConcurrentHashMap<>();
 
   public CloseableThriftHiveMetastoreIfaceClientFactory(
       TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory,
@@ -96,8 +99,9 @@ private CloseableThriftHiveMetastoreIface newHiveInstance(
     }
     properties.put(ConfVars.METASTOREURIS.varname, uris);
     HiveConfFactory confFactory = new HiveConfFactory(Collections.emptyList(), properties);
+    HiveConf hiveConf = cachedHiveConf.computeIfAbsent(uris, t-> confFactory.newInstance());
     return defaultMetaStoreClientFactory
-        .newInstance(confFactory.newInstance(), "waggledance-" + name, DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY,
+        .newInstance(hiveConf, "waggledance-" + name, DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY,
             connectionTimeout);
   }