Merge pull request #321 from flaming-archer/hive-3.x-backport-main

Hive 3.x backport  readonly and ratelimit feature

.github/workflows/release.yml

@@ -0,0 +1,60 @@
+name: Release to Maven Central
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch to use to release from."
+        required: true
+        default: "main"
+jobs:
+  release:
+    name: Release to Maven Central
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        ref: ${{ github.event.inputs.branch }}
+        # We need a personal access token to be able to push to a protected branch
+        token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+    - name: Set up JDK
+      uses: actions/setup-java@v2
+      with:
+        distribution: 'adopt'
+        java-version: '8'
+        java-package: jdk
+        server-id: sonatype-nexus-staging # Value of the distributionManagement/repository/id field of the pom.xml
+        server-username: SONATYPE_USERNAME # env variable for username in deploy
+        server-password: SONATYPE_PASSWORD # env variable for token in deploy
+        # only signed artifacts will be released to maven central. this sets up things for the maven-gpg-plugin
+        gpg-private-key: ${{ secrets.HCOM_GPG_PRIVATE_KEY }} # Value of the GPG private key to import
+        gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase
+        # this creates a settings.xml with the following server
+        settings-path: ${{ github.workspace }}
+
+    - name: Configure Git User
+      run: |
+        git config user.email "oss@expediagroup.com"
+        git config user.name "eg-oss-ci"
+
+    - name: Install Local Dependencies
+      run: |
+        mvn install:install-file -Dfile=lib/aws-glue-datacatalog-hive-client-1.10.0-WD.pom -DpomFile=lib/aws-glue-datacatalog-hive-client-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/shims-1.10.0-WD.pom -DpomFile=lib/shims-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/shims-common-1.10.0-WD.jar -DpomFile=lib/shims-common-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/hive2-shims-1.10.0-WD.jar -DpomFile=lib/hive2-shims-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/spark-hive-shims-1.10.0-WD.jar -DpomFile=lib/spark-hive-shims-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/shims-loader-1.10.0-WD.jar -DpomFile=lib/shims-loader-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/aws-glue-datacatalog-client-common-1.10.0-WD.jar -DpomFile=lib/aws-glue-datacatalog-client-common-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/aws-glue-datacatalog-hive2-client-1.10.0-WD.jar -DpomFile=lib/aws-glue-datacatalog-hive2-client-1.10.0-WD.pom
+
+    - name: Run Maven Targets
+      # we are skipping the tests for releases, run a build step first.
+      run: mvn release:prepare release:perform --settings $GITHUB_WORKSPACE/settings.xml -DskipTests -Darguments=-DskipTests --activate-profiles sonatype-oss-release-github-actions --batch-mode --show-version --no-transfer-progress
+      env:
+        SONATYPE_PASSWORD: ${{ secrets.HCOM_SONATYPE_PASSWORD }}
+        SONATYPE_USERNAME: ${{ secrets.HCOM_SONATYPE_USERNAME }}
+        GPG_PASSPHRASE: ${{secrets.HCOM_GPG_PRIVATE_KEY_PASSPHRASE}}

CHANGELOG.md

@@ -13,6 +13,17 @@
 * `lombok` - `1.18.24`.
 * `jakarta` - `6.0.0`.
 * `apache-commons` - `3.12.0`.
+## [3.13.0] - 2024-04-19
+### Added
+- Added `waggle-dance-extensions` module. See [extensions README](waggle-dance-extensions/README.md.)
+- Added support to enable Rate Limiting in Waggle Dance.
+### Changed
+- Changed and added some log messages for better tracking of calls.
+- Changed Integration Test WaggleDanceRunner to allow for reuse.
+
+## [3.12.0] - 2024-02-08
+### Added
+- Added optional `primary-meta-store.read-only-remote-meta-store-uris` config to allow traffic to be diverted based on calls made. See README.md.
 
 ### Fixed
 * Added lombok

README.md

@@ -112,6 +112,8 @@ The table below describes all the available configuration values for Waggle Danc
 | `status-polling-delay-time-unit`  | No         | Controls the delay time unit. Default is `MINUTES` . |
 | `configuration-properties`        | No         | Map of Hive properties that will be added to the HiveConf used when creating the Thrift clients (they will be shared among all the clients). |
 
+Extensions (for instance Rate Limiting) are described here: [waggle-dance-extensions/README.md](waggle-dance-extensions/README.md)
+
 ### Federation
 
 Federation config is by default located in: `$WAGGLE_DANCE_HOME/conf/waggle-dance-federation.yml`.
@@ -170,6 +172,7 @@ The table below describes all the available configuration values for Waggle Danc
 | `primary-meta-store.mapped-tables`                       | No       | List of mappings from databases to tables to federate from the primary metastore, similar to `mapped-databases`. By default, all tables are available. See `mapped-tables` configuration below. |
 | `primary-meta-stores.hive-metastore-filter-hook`         | No       | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. |
 | `primary-meta-stores.database-name-mapping`              | No       | BiDirectional Map of database names and mapped name, where key=`<database name as known in the primary metastore>` and value=`<name that should be shown to a client>`. See the [Database Name Mapping](#database-name-mapping) section.|
+| `primary-meta-store.read-only-remote-meta-store-uris`    | No       | Can be used to configure an extra read-only endpoint for the primary Metastore. This is an optimization if your environment runs separate Metastore endpoints and traffic needs to be diverted efficiently. Waggle Dance will direct traffic to the read-write or read-only endpoints based on the call being done. For instance `get_table` will be a read-only call but `alter_table` will be forwarded to the read-write Metastore.| 
 | `primary-meta-stores.configuration-properties`           | No       | Map of the primary metastore personalized properties that will be added to the HiveConf used when creating the Thrift clients (they will be effect only on this client),the priority is higher than the properites of the same name in waggle-dance-server.yml.                                                                                                                                                                                                                                                                                                                         |
 | `federated-meta-stores`                                  | No       | Possible empty list of read only federated metastores. |
 | `federated-meta-stores[n].remote-meta-store-uris`        | Yes      | Thrift URIs of the federated read-only metastore. |
@@ -576,4 +579,4 @@ The Waggle Dance logo uses the [Beetype Filled font](http://www.1001fonts.com/be
 ## Legal
 This project is available under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html).
 
-Copyright 2016-2019 Expedia, Inc.
+Copyright 2016-2024 Expedia, Inc.

pom.xml

@@ -24,6 +24,7 @@
   </scm>
 
   <modules>
+    <module>waggle-dance-extensions</module>
     <module>waggle-dance-core</module>
     <module>waggle-dance-api</module>
     <module>waggle-dance-rest</module>

waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java

@@ -28,8 +28,6 @@
 import javax.validation.constraints.NotBlank;
 import javax.validation.constraints.NotNull;
 
-import lombok.NoArgsConstructor;
-
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
@@ -45,7 +43,6 @@
 @JsonSubTypes({
     @Type(value = PrimaryMetaStore.class, name = "PRIMARY"),
     @Type(value = FederatedMetaStore.class, name = "FEDERATED") })
-@NoArgsConstructor
 public abstract class AbstractMetaStore {
   private String databasePrefix;
   private String hiveMetastoreFilterHook;
@@ -62,6 +59,9 @@ public abstract class AbstractMetaStore {
   private transient @JsonIgnore HashBiMap<String, String> databaseNameBiMapping = HashBiMap.create();
   private boolean impersonationEnabled;
   private Map<String, String> configurationProperties = new HashMap<>();
+  private String readOnlyRemoteMetaStoreUris;
+
+  public AbstractMetaStore() {}
 
   public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) {
     this.name = name;
@@ -127,6 +127,14 @@ public void setRemoteMetaStoreUris(String remoteMetaStoreUris) {
     this.remoteMetaStoreUris = remoteMetaStoreUris;
   }
 
+  public String getReadOnlyRemoteMetaStoreUris() {
+    return readOnlyRemoteMetaStoreUris;
+  }
+
+  public void setReadOnlyRemoteMetaStoreUris(String readOnlyRemoteMetaStoreUris) {
+    this.readOnlyRemoteMetaStoreUris = readOnlyRemoteMetaStoreUris;
+  }
+
   public MetastoreTunnel getMetastoreTunnel() {
     return metastoreTunnel;
   }
@@ -256,6 +264,7 @@ public String toString() {
         .add("databasePrefix", databasePrefix)
         .add("federationType", getFederationType())
         .add("remoteMetaStoreUris", remoteMetaStoreUris)
+        .add("readOnlyRemoteMetaStoreUris", readOnlyRemoteMetaStoreUris)
         .add("metastoreTunnel", metastoreTunnel)
         .add("accessControlType", accessControlType)
         .add("writableDatabaseWhiteList", writableDatabaseWhitelist)

waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java

@@ -72,7 +72,7 @@ public void nullDatabasePrefix() {
 
   @Test
   public void toJson() throws Exception {
-    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
+    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"readOnlyRemoteMetaStoreUris\":null,\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
     ObjectMapper mapper = new ObjectMapper();
     // Sorting to get deterministic test behaviour
     mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);

waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java

@@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() {
 
   @Test
   public void toJson() throws Exception {
-    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
+    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"readOnlyRemoteMetaStoreUris\":null,\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
     ObjectMapper mapper = new ObjectMapper();
     // Sorting to get deterministic test behaviour
     mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);

waggle-dance-boot/pom.xml

@@ -25,6 +25,11 @@
       <artifactId>waggle-dance-rest</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.hotels</groupId>
+      <artifactId>waggle-dance-extensions</artifactId>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

waggle-dance-core/pom.xml

@@ -140,6 +140,22 @@
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-exec</artifactId>
+      <version>${hive.version}</version>
+      <classifier>core</classifier>
+      <exclusions>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.pentaho</groupId>
+          <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.calcite.avatica</groupId>
+          <artifactId>avatica</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
@@ -154,6 +170,22 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-runner</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty.aggregate</groupId>
+          <artifactId>jetty-all</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.pentaho</groupId>
+          <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-exec</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -252,6 +284,12 @@
       <groupId>com.hotels</groupId>
       <artifactId>beeju</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-exec</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>fm.last.commons</groupId>

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/WaggleDance.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIface.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java

@@ -25,23 +25,32 @@
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 
-import lombok.AllArgsConstructor;
-
 import com.hotels.bdp.waggledance.api.model.AbstractMetaStore;
 import com.hotels.bdp.waggledance.client.tunnelling.TunnelingMetaStoreClientFactory;
 import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration;
 import com.hotels.bdp.waggledance.context.CommonBeans;
 import com.hotels.hcommon.hive.metastore.conf.HiveConfFactory;
 import com.hotels.hcommon.hive.metastore.util.MetaStoreUriNormaliser;
 
-@AllArgsConstructor
-public class CloseableThriftHiveMetastoreIfaceClientFactory {
+public class CloseableThriftHiveMetastoreIfaceClientFactory implements ThriftClientFactory {
 
   private static final int DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY = 3;
   private final TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory;
   private final DefaultMetaStoreClientFactory defaultMetaStoreClientFactory;
   private final WaggleDanceConfiguration waggleDanceConfiguration;
   private final int defaultConnectionTimeout = (int) TimeUnit.SECONDS.toMillis(2L);
+  private final SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory;
+
+  public CloseableThriftHiveMetastoreIfaceClientFactory(
+      TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory,
+      DefaultMetaStoreClientFactory defaultMetaStoreClientFactory,
+      WaggleDanceConfiguration waggleDanceConfiguration,
+      SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory) {
+    this.tunnelingMetaStoreClientFactory = tunnelingMetaStoreClientFactory;
+    this.defaultMetaStoreClientFactory = defaultMetaStoreClientFactory;
+    this.waggleDanceConfiguration = waggleDanceConfiguration;
+    this.splitTrafficMetaStoreClientFactory = splitTrafficMetaStoreClientFactory;
+  }
 
   public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore) {
     Map<String, String> properties = new HashMap<>();
@@ -51,16 +60,24 @@ public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore
     if (metaStore.getConfigurationProperties() != null) {
       properties.putAll(metaStore.getConfigurationProperties());
     }
+    String name = metaStore.getName().toLowerCase(Locale.ROOT);
+    if (metaStore.getReadOnlyRemoteMetaStoreUris() != null) {
+      CloseableThriftHiveMetastoreIface readWrite = newHiveInstance(metaStore, name, metaStore.getRemoteMetaStoreUris(),
+          properties);
+      CloseableThriftHiveMetastoreIface readOnly = newHiveInstance(metaStore, name + "_ro",
+          metaStore.getReadOnlyRemoteMetaStoreUris(), properties);
+      return splitTrafficMetaStoreClientFactory.newInstance(readWrite, readOnly);
 
-    return newHiveInstance(metaStore, properties);
+    }
+    return newHiveInstance(metaStore, name, metaStore.getRemoteMetaStoreUris(), properties);
   }
 
   private CloseableThriftHiveMetastoreIface newHiveInstance(
-          AbstractMetaStore metaStore,
-          Map<String, String> properties) {
-    String uris = MetaStoreUriNormaliser.normaliseMetaStoreUris(metaStore.getRemoteMetaStoreUris());
-    String name = metaStore.getName().toLowerCase(Locale.ROOT);
-
+      AbstractMetaStore metaStore,
+      String name,
+      String metaStoreUris,
+      Map<String, String> properties) {
+    String uris = MetaStoreUriNormaliser.normaliseMetaStoreUris(metaStoreUris);
     // Connection timeout should not be less than 1
     // A timeout of zero is interpreted as an infinite timeout, so this is avoided
     int connectionTimeout = Math.max(1, defaultConnectionTimeout + (int) metaStore.getLatency());