Merge branch 'master' into fix_optimizer_table_loading

amoro-ams/amoro-ams-dashboard/src/types/common.type.ts

@@ -242,6 +242,7 @@ export interface IOptimizeTableItem {
   quotaOccupationDesc: string
   duration: number
   durationDesc: string
+  durationDisplay: string
   fileSizeDesc: string
   tableIdentifier: ITableIdentifier
   tableNameOnly?: string

amoro-ams/amoro-ams-dashboard/src/utils/index.ts

@@ -57,51 +57,33 @@ export function mbToSize(size: number): string {
 /**
  * Convert ms to d h min s
  */
-export function formatMS2Time(time: number, fromHour?: boolean): string {
-  if (time === null || time === undefined || Number.isNaN(time)) {
-    return '-'
-  }
-  // 3h 34min 12s
-  const Second = 1000
-  const Minute = Second * 60
-  const Hour = Minute * 60
-  const Day = Hour * 24
-  if (time === 0) {
-    return '0 ms'
-  }
-  if (time < Second) {
-    return fromHour ? '0 min ' : `${time} ms`
-  }
-  if (time >= Second && time < Minute) {
-    const s = Math.floor(time / Second)
-    if (fromHour) {
-      return '0 min'
-    }
-    return s ? `${s} s` : ''
-  }
-  if (time >= Minute && time < Hour) {
-    const calcMin = Math.floor(time / Minute)
-    const s = Math.floor((time - calcMin * Minute) / Second)
-    if (fromHour) {
-      return `${calcMin} min`
-    }
-    else {
-      return time % Minute === 0 ? `${time / Minute} min` : `${calcMin} min ${s ? `${s} s` : ''}`
-    }
-  }
-  if (time % Hour === 0) {
-    return `${time / Hour} h`
-  }
-  if (time >= Hour && time < Day) {
-    const calcHour = Math.floor(time / Hour)
-    return `${calcHour} h ${formatMS2Time(time - calcHour * Hour, true)}`
-  }
-  if (time % Day === 0) {
-    return `${time / Day} d`
-  }
+export function formatMS2Time(ms: number): string {
+  const secondsInMs = 1000
+  const minutesInMs = secondsInMs * 60
+  const hoursInMs = minutesInMs * 60
+  const daysInMs = hoursInMs * 24
+
+  const days = Math.floor(ms / daysInMs)
+  ms %= daysInMs
+
+  const hours = Math.floor(ms / hoursInMs)
+  ms %= hoursInMs
+
+  const minutes = Math.floor(ms / minutesInMs)
+  ms %= minutesInMs
+
+  const seconds = Math.floor(ms / secondsInMs)
+
+  const times = [
+    { value: days, unit: 'd' },
+    { value: hours, unit: 'h' },
+    { value: minutes, unit: 'min' },
+    { value: seconds, unit: 's' },
+  ]
 
-  const calcDay = Math.floor(time / Day)
-  return `${calcDay} d ${formatMS2Time(time - calcDay * Day, true)}`
+  return times.reduce((pre: string, cur: { value: number, unit: string }) => {
+    return cur.value > 0 ? `${pre}${cur.value} ${cur.unit} ` : pre
+  }, '')
 }
 /**
  * Convert milliseconds to d h min s format

amoro-ams/amoro-ams-dashboard/src/utils/request.ts

@@ -23,7 +23,7 @@ import useStore from '@/store'
 import router from '@/router'
 
 // http://10.196.98.23:19111/
-export const baseURL = import.meta.env.MODE === 'mock' ? '/mock' : '/'
+export const baseURL = import.meta.env.PROD ? '' : (import.meta.env.MODE === 'mock' ? '/mock' : '/')
 
 // Used to store request identification information whose current status is pending
 export const pendingRequest: {

amoro-ams/amoro-ams-dashboard/src/views/optimize/components/List.vue

@@ -91,8 +91,8 @@ async function getTableList() {
     pagination.total = total;
     (list || []).forEach((p: IOptimizeTableItem) => {
       p.quotaOccupationDesc = p.quotaOccupation - 0.0005 > 0 ? `${(p.quotaOccupation * 100).toFixed(1)}%` : '0'
-      p.durationDesc = formatMS2Time(p.duration || 0) as string
-      (p as any).durationDisplay = formatMS2DisplayTime(p.duration || 0)
+      p.durationDesc = p.duration ? formatMS2Time(p.duration) : '-'
+      p.durationDisplay = formatMS2DisplayTime(p.duration || 0)
       p.fileSizeDesc = bytesToSize(p.fileSize)
       dataSource.push(p)
     })

amoro-ams/amoro-ams-dashboard/src/views/resource/components/List.vue

@@ -197,13 +197,13 @@ const groupRecord = ref<IIOptimizeGroupItem>({
   container: '',
   resourceOccupation: '',
 })
-const scaleOutViseble = ref<boolean>(false)
+const scaleOutVisible = ref<boolean>(false)
 function scaleOutGroup(record: IIOptimizeGroupItem) {
   if (record.container === 'external') {
     return
   }
   groupRecord.value = { ...record }
-  scaleOutViseble.value = true
+  scaleOutVisible.value = true
 }
 
 function changeTable({ current = pagination.current, pageSize = pagination.pageSize }) {
@@ -221,7 +221,10 @@ onMounted(() => {
 <template>
   <div class="list-wrap">
     <a-table
-      class="ant-table-common" :columns="columns" :data-source="dataSource" :pagination="pagination"
+      class="ant-table-common"
+      :columns="columns"
+      :data-source="dataSource"
+      :pagination="pagination"
       :loading="loading" @change="changeTable"
     >
       <template #bodyCell="{ column, record }">
@@ -262,7 +265,7 @@ onMounted(() => {
       </template>
     </a-table>
   </div>
-  <ScaleOut v-if="scaleOutViseble" :group-record="groupRecord" @cancel="scaleOutViseble = false" @refresh="refresh" />
+  <ScaleOut v-if="scaleOutVisible" :group-record="groupRecord" @cancel="scaleOutVisible = false" @refresh="refresh" />
   <u-loading v-if="releaseLoading" />
 </template>
 

amoro-ams/amoro-ams-dashboard/src/views/tables/components/Optimizing.vue

@@ -107,7 +107,7 @@ async function refreshOptimizingProcesses() {
         startTime: item.startTime ? dateFormat(item.startTime) : '-',
         finishTime: item.finishTime ? dateFormat(item.finishTime) : '-',
         optimizingType: item.optimizingType ? item.optimizingType : '-',
-        duration: formatMS2Time(item.duration || '-'),
+        duration: item.duration ? formatMS2Time(item.duration) : '-',
         inputFiles: `${bytesToSize(inputFiles.totalSize)} / ${inputFiles.fileCnt}`,
         outputFiles: `${bytesToSize(outputFiles.totalSize)} / ${outputFiles.fileCnt}`,
         tasks: `${item.successTasks || '0'} / ${item.totalTasks || '0'}${item.runningTasks ? ` (${item.runningTasks} running)` : ''}`,
@@ -186,7 +186,7 @@ async function refreshOptimizingTasks() {
     list.forEach((p: BreadcrumbOptimizingItem) => {
       p.startTime = p.startTime ? dateFormat(p.startTime) : '-'
       p.endTime = p.endTime ? dateFormat(p.endTime) : '-'
-      p.formatCostTime = formatMS2Time(p.costTime)
+      p.formatCostTime = p.costTime ? formatMS2Time(p.costTime) : '-'
       p.thread = p.optimizerToken ? `(${p.threadId})${p.optimizerToken}` : '-'
       p.partitionData = p.partitionData ? p.partitionData : '-'
       p.inputFilesDesc = `${bytesToSize(p.inputFiles.totalSize)} / ${p.inputFiles.fileCnt}`

amoro-ams/amoro-ams-dashboard/src/views/tables/index.vue

@@ -193,7 +193,7 @@ export default defineComponent({
     font-size: 24px;
     line-height: 1.5;
     margin-right: 16px;
-    max-width: 600px;
+    max-width: 100%;
     padding-left: 24px;
   }
   .table-info {

amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/dashboard/model/SqlExample.java

@@ -27,7 +27,7 @@ public enum SqlExample {
           + "    name string, \n"
           + "    ts timestamp,\n"
           + "    primary key (id)\n"
-          + ") using arctic \n"
+          + ") using mixed_iceberg \n"
           + "partitioned by (days(ts)) \n"
           + "tblproperties ('table.props' = 'val');"),
   DELETE_TABLE("DeleteTable", "drop table db_name.table_name;"),

amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java

@@ -103,6 +103,12 @@ public class IcebergTableMaintainer implements TableMaintainer {
   public static final String EXPIRE_TIMESTAMP_MS = "TIMESTAMP_MS";
   public static final String EXPIRE_TIMESTAMP_S = "TIMESTAMP_S";
 
+  public static final Set<String> AMORO_MAINTAIN_COMMITS =
+      Sets.newHashSet(
+          CommitMetaProducer.OPTIMIZE.name(),
+          CommitMetaProducer.DATA_EXPIRATION.name(),
+          CommitMetaProducer.CLEAN_DANGLING_DELETE.name());
+
   protected Table table;
 
   public IcebergTableMaintainer(Table table) {
@@ -435,6 +441,9 @@ private int clearInternalTableDanglingDeleteFiles() {
         Collections.emptySet(),
         Collections.emptySet());
     try {
+      rewriteFiles.set(
+          org.apache.amoro.op.SnapshotSummary.SNAPSHOT_PRODUCER,
+          CommitMetaProducer.CLEAN_DANGLING_DELETE.name());
       rewriteFiles.commit();
     } catch (ValidationException e) {
       LOG.warn("Iceberg RewriteFiles commit failed on clear danglingDeleteFiles, but ignore", e);
@@ -498,10 +507,7 @@ public static Snapshot findLatestSnapshotContainsKey(Table table, String summary
   public static long fetchLatestNonOptimizedSnapshotTime(Table table) {
     Optional<Snapshot> snapshot =
         IcebergTableUtil.findFirstMatchSnapshot(
-            table,
-            s ->
-                !s.summary().containsValue(CommitMetaProducer.OPTIMIZE.name())
-                    && !s.summary().containsValue(CommitMetaProducer.DATA_EXPIRATION.name()));
+            table, s -> s.summary().values().stream().noneMatch(AMORO_MAINTAIN_COMMITS::contains));
     return snapshot.map(Snapshot::timestampMillis).orElse(Long.MAX_VALUE);
   }