BREAKING CHANGE:glue_sync skip archive by default (#106)

* BREAKING CHANGE:glue_sync skip archive by default

CHANGELOG.md

@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 8.0.0 - 2024-08-27
+### Changed
+- `ApiaryGlueSync` override `skipArchive` to be `true` by default. Backward incompatible behavior that turns off table archiving by default. Allows for per table overrides if needed. If you don't rely on Glue table version it is safe to upgrade to this version of the glue listener without making changes.
+
 ## 7.3.11 - 2024-06-19
 ### Changed
 - Upgrade aws version from `1.11.520` to `1.12.276` in `apiary-receiver-sqs`.

apiary-metastore-events/apiary-hive-events/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-metastore-events-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-hive-events</artifactId>

apiary-metastore-events/kafka-metastore-events/kafka-metastore-integration-tests/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>kafka-metastore-events-parent</artifactId>
     <groupId>com.expediagroup.apiary</groupId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>kafka-metastore-integration-tests</artifactId>

apiary-metastore-events/kafka-metastore-events/kafka-metastore-listener/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>kafka-metastore-events-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>kafka-metastore-listener</artifactId>

apiary-metastore-events/kafka-metastore-events/kafka-metastore-listener/src/main/java/com/expediagroup/apiary/extensions/events/metastore/kafka/messaging/KafkaMessageSender.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2018-2020 Expedia, Inc.
+ * Copyright (C) 2018-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

apiary-metastore-events/kafka-metastore-events/kafka-metastore-listener/src/main/java/com/expediagroup/apiary/extensions/events/metastore/kafka/messaging/KafkaProducerProperty.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2018-2020 Expedia, Inc.
+ * Copyright (C) 2018-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

apiary-metastore-events/kafka-metastore-events/kafka-metastore-listener/src/test/java/com/expediagroup/apiary/extensions/events/metastore/kafka/messaging/KafkaProducerPropertyTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2018-2020 Expedia, Inc.
+ * Copyright (C) 2018-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

apiary-metastore-events/kafka-metastore-events/kafka-metastore-receiver/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>kafka-metastore-events-parent</artifactId>
     <groupId>com.expediagroup.apiary</groupId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>kafka-metastore-receiver</artifactId>

apiary-metastore-events/kafka-metastore-events/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-metastore-events-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>kafka-metastore-events-parent</artifactId>

apiary-metastore-events/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-extensions-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-metastore-events-parent</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-metastore-consumers/metastore-consumer-common/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-metastore-consumers-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>metastore-consumer-common</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-metastore-consumers/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>sns-metastore-events-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-metastore-consumers-parent</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-metastore-consumers/privileges-grantor/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-metastore-consumers-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-privileges-grantor-parent</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-metastore-consumers/privileges-grantor/privileges-grantor-core/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-privileges-grantor-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-privileges-grantor-core</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-metastore-consumers/privileges-grantor/privileges-grantor-lambda/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-privileges-grantor-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-privileges-grantor-lambda</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-metastore-listener/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>sns-metastore-events-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-metastore-listener</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-receivers/apiary-receiver-common/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-receivers-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-receiver-common</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-receivers/apiary-receiver-sqs/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-receivers-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-receiver-sqs</artifactId>

apiary-metastore-events/sns-metastore-events/apiary-receivers/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>sns-metastore-events-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-receivers-parent</artifactId>

apiary-metastore-events/sns-metastore-events/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-metastore-events-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>sns-metastore-events-parent</artifactId>

apiary-metastore-metrics/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>apiary-extensions-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-metastore-metrics</artifactId>

hive-event-listeners/apiary-gluesync-listener/README.md

@@ -14,6 +14,11 @@ The GlueSync listener can be configured by setting the following System Environm
 |----|----|----|
 GLUE_PREFIX|No|Prefix added to Glue databases to handle database name collisions when synchronizing multiple metastores to the Glue catalog.
 
+## Table update SkipArchive
+[AWS default](https://docs.aws.amazon.com/glue/latest/webapi/API_UpdateTable.html#Glue-UpdateTable-request-SkipArchive) is to archive the table on every update. This especially with Iceberg tables can lead to a lot of table version of which you can only have a certain limit. To counter this we override this property and set skipArchive=true so do *not* make an archive of the table when updating. 
+If an archive is needed, this can be done per table by setting the Hive table property: 'apiary.gluesync.skipArchive=false'.
+
+
 # Legal
 This project is available under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html).
 

hive-event-listeners/apiary-gluesync-listener/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>hive-event-listeners-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-gluesync-listener</artifactId>

hive-event-listeners/apiary-gluesync-listener/src/main/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSync.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2018-2022 Expedia, Inc.
+ * Copyright (C) 2018-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@
  */
 package com.expediagroup.apiary.extensions.gluesync.listener;
 
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
@@ -65,6 +64,8 @@
 
 public class ApiaryGlueSync extends MetaStoreEventListener {
 
+  static final String APIARY_GLUESYNC_SKIP_ARCHIVE_TABLE_PARAM = "apiary.gluesync.skipArchive";
+
   private static final Logger log = LoggerFactory.getLogger(ApiaryGlueSync.class);
 
   private static final String MANAGED_BY_GLUESYNC_KEY = "managed-by";
@@ -179,7 +180,9 @@ public void onAlterTable(AlterTableEvent event) {
     }
     Table table = event.getNewTable();
     try {
+      boolean skipArchive = shouldSkipArchive(table);
       UpdateTableRequest updateTableRequest = new UpdateTableRequest()
+          .withSkipArchive(skipArchive)
           .withTableInput(transformTable(table))
           .withDatabaseName(glueDbName(table));
       glueClient.updateTable(updateTableRequest);
@@ -194,6 +197,18 @@ public void onAlterTable(AlterTableEvent event) {
     }
   }
 
+  private boolean shouldSkipArchive(Table table) {
+    boolean skipArchive = true;
+    if (table.getParameters() != null) {
+      //Only if explicitly overridden to false do enable table archive. Normally we want to skip archiving. 
+      String skipArchiveParam = table.getParameters().get(APIARY_GLUESYNC_SKIP_ARCHIVE_TABLE_PARAM);
+      if ("false".equals(skipArchiveParam)) {
+        skipArchive = false;
+      }
+    }
+    return skipArchive;
+  }
+
   @Override
   public void onAddPartition(AddPartitionEvent event) {
     if (!event.getStatus()) {

hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/ApiaryGlueSyncTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2018-2022 Expedia, Inc.
+ * Copyright (C) 2018-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@
 
 import static com.google.common.collect.Maps.newHashMap;
 
+import static com.expediagroup.apiary.extensions.gluesync.listener.ApiaryGlueSync.APIARY_GLUESYNC_SKIP_ARCHIVE_TABLE_PARAM;
 import static com.expediagroup.apiary.extensions.gluesync.listener.IcebergTableOperations.simpleIcebergPartitionSpec;
 import static com.expediagroup.apiary.extensions.gluesync.listener.IcebergTableOperations.simpleIcebergSchema;
 import static com.expediagroup.apiary.extensions.gluesync.listener.IcebergTableOperations.simpleIcebergTable;
@@ -252,6 +253,32 @@ public void onAlterHiveTable() {
     assertThat(updateTableRequest.getTableInput().getLastAccessTime(), is(new Date(lastAccessTime)));
     assertThat(toList(updateTableRequest.getTableInput().getPartitionKeys()), is(asList(partNames)));
     assertThat(toList(updateTableRequest.getTableInput().getStorageDescriptor().getColumns()), is(asList(colNames)));
+    assertThat(updateTableRequest.getSkipArchive(), is(true));
+  }
+
+  @Test
+  public void onAlterHiveTableSkipArchiveOverride() {
+    AlterTableEvent event = mock(AlterTableEvent.class);
+    when(event.getStatus()).thenReturn(true);
+
+    Table newTable = simpleHiveTable(simpleSchema(), simplePartitioning());
+    int lastAccessTime = 10000000;
+    newTable.setLastAccessTime(lastAccessTime);
+    newTable.setTableName("table2");
+    newTable.putToParameters(APIARY_GLUESYNC_SKIP_ARCHIVE_TABLE_PARAM, "false");
+    when(event.getNewTable()).thenReturn(newTable);
+
+    glueSync.onAlterTable(event);
+
+    verify(glueClient).updateTable(updateTableRequestCaptor.capture());
+    UpdateTableRequest updateTableRequest = updateTableRequestCaptor.getValue();
+
+    assertThat(updateTableRequest.getDatabaseName(), is(gluePrefix + dbName));
+    assertThat(updateTableRequest.getTableInput().getName(), is("table2"));
+    assertThat(updateTableRequest.getTableInput().getLastAccessTime(), is(new Date(lastAccessTime)));
+    assertThat(toList(updateTableRequest.getTableInput().getPartitionKeys()), is(asList(partNames)));
+    assertThat(toList(updateTableRequest.getTableInput().getStorageDescriptor().getColumns()), is(asList(colNames)));
+    assertThat(updateTableRequest.getSkipArchive(), is(false));
   }
 
   @Test

hive-event-listeners/apiary-gluesync-listener/src/test/java/com/expediagroup/apiary/extensions/gluesync/listener/IcebergTableOperations.java

@@ -1,3 +1,18 @@
+/**
+ * Copyright (C) 2018-2022 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.expediagroup.apiary.extensions.gluesync.listener;
 
 import static org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE;

hive-event-listeners/apiary-metastore-auth/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>hive-event-listeners-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-metastore-auth</artifactId>

hive-event-listeners/apiary-ranger-metastore-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.expediagroup.apiary</groupId>
     <artifactId>hive-event-listeners-parent</artifactId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>apiary-ranger-metastore-plugin</artifactId>

hive-event-listeners/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <artifactId>apiary-extensions-parent</artifactId>
     <groupId>com.expediagroup.apiary</groupId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>hive-event-listeners-parent</artifactId>

hive-hooks/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <artifactId>apiary-extensions-parent</artifactId>
     <groupId>com.expediagroup.apiary</groupId>
-    <version>7.3.12-SNAPSHOT</version>
+    <version>8.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>hive-hooks</artifactId>

pom.xml

@@ -10,7 +10,7 @@
   <groupId>com.expediagroup.apiary</groupId>
   <artifactId>apiary-extensions-parent</artifactId>
   <description>Various extensions to Apiary that provide additional, optional functionality</description>
-  <version>7.3.12-SNAPSHOT</version>
+  <version>8.0.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Apiary Extensions Parent</name>
   <inceptionYear>2018</inceptionYear>