Add the "clean-orphan-file.interval-time-minutes" parameter

apache · Sep 10, 2024 · 2d92ecb · 2d92ecb
1 parent 62a095f
commit 2d92ecb
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 3 deletions.
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java
@@ -91,6 +91,11 @@ public static TableConfiguration parseTableConfig(Map<String, String> properties
                 properties,
                 TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME,
                 TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT))
+        .setCleanOrphanIntervalMinutes(
+            CompatiblePropertyUtil.propertyAsLong(
+                properties,
+                TableProperties.ORPHAN_FILE_CLEAN_INTERVAL_TIME,
+                TableProperties.ORPHAN_FILE_CLEAN_INTERVAL_TIME_DEFAULT))
         .setDeleteDanglingDeleteFilesEnabled(
             gcEnabled
                 && CompatiblePropertyUtil.propertyAsBoolean(

diff --git a/...ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java b/...ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java
@@ -29,15 +29,13 @@
 public class OrphanFilesCleaningExecutor extends BaseTableExecutor {
   private static final Logger LOG = LoggerFactory.getLogger(OrphanFilesCleaningExecutor.class);
 
-  private static final long INTERVAL = 24 * 60 * 60 * 1000L;
-
   public OrphanFilesCleaningExecutor(TableManager tableRuntimes, int poolSize) {
     super(tableRuntimes, poolSize);
   }
 
   @Override
   protected long getNextExecutingTime(TableRuntime tableRuntime) {
-    return INTERVAL;
+    return tableRuntime.getTableConfiguration().getCleanOrphanIntervalMinutes() * 60 * 1000;
   }
 
   @Override

diff --git a/amoro-common/src/main/java/org/apache/amoro/config/TableConfiguration.java b/amoro-common/src/main/java/org/apache/amoro/config/TableConfiguration.java
@@ -31,6 +31,7 @@ public class TableConfiguration {
   private long changeDataTTLMinutes;
   private boolean cleanOrphanEnabled;
   private long orphanExistingMinutes;
+  private long cleanOrphanIntervalMinutes;
   private boolean deleteDanglingDeleteFilesEnabled;
   private OptimizingConfig optimizingConfig;
   private DataExpirationConfig expiringDataConfig;
@@ -62,6 +63,10 @@ public long getOrphanExistingMinutes() {
     return orphanExistingMinutes;
   }
 
+  public long getCleanOrphanIntervalMinutes() {
+    return cleanOrphanIntervalMinutes;
+  }
+
   public OptimizingConfig getOptimizingConfig() {
     return optimizingConfig;
   }
@@ -101,6 +106,11 @@ public TableConfiguration setOrphanExistingMinutes(long orphanExistingMinutes) {
     return this;
   }
 
+  public TableConfiguration setCleanOrphanIntervalMinutes(long cleanOrphanIntervalMinutes) {
+    this.cleanOrphanIntervalMinutes = cleanOrphanIntervalMinutes;
+    return this;
+  }
+
   public boolean isDeleteDanglingDeleteFilesEnabled() {
     return deleteDanglingDeleteFilesEnabled;
   }

diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java
@@ -173,6 +173,11 @@ private TableProperties() {}
   public static final String MIN_ORPHAN_FILE_EXISTING_TIME =
       "clean-orphan-file.min-existing-time-minutes";
   public static final long MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT = 2880; // 2 Days
+
+  public static final String ORPHAN_FILE_CLEAN_INTERVAL_TIME =
+      "clean-orphan-file.interval-time-minutes";
+  public static final long ORPHAN_FILE_CLEAN_INTERVAL_TIME_DEFAULT = 4320; // 3 Days
+
   public static final String ENABLE_DANGLING_DELETE_FILES_CLEAN =
       "clean-dangling-delete-files.enabled";
   public static final boolean ENABLE_DANGLING_DELETE_FILES_CLEAN_DEFAULT = true;

diff --git a/docs/user-guides/configurations.md b/docs/user-guides/configurations.md
@@ -73,6 +73,7 @@ Data-cleaning configurations are applicable to both Iceberg Format and Mixed str
 | snapshot.keep.min-count                     | 1                | Minimum number of snapshots retained for table expiration                                                                                                                                                                                                             |
 | clean-orphan-file.enabled                   | false            | Enables periodically clean orphan files                                                                                                                                                                                                                               |
 | clean-orphan-file.min-existing-time-minutes | 2880(2 days)     | Cleaning orphan files keeps the files modified within a specified time in minutes                                                                                                                                                                                     |
+| clean-orphan-file.interval-time-minutes     | 4320(3 days)     | The interval for Cleaning orphan files within a specified time in minutes                                                                                                                                                                                             |
 | clean-dangling-delete-files.enabled         | true             | Whether to enable cleaning of dangling delete files                                                                                                                                                                                                                   |
 | data-expire.enabled                         | false            | Whether to enable data expiration                                                                                                                                                                                                                                     |
 | data-expire.level                           | partition        | Level of data expiration. Including partition and file                                                                                                                                                                                                                |