diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java index b83be317b4..43a71e785d 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java @@ -48,6 +48,7 @@ import org.apache.amoro.server.dashboard.utils.AmsUtil; import org.apache.amoro.server.dashboard.utils.CommonUtil; import org.apache.amoro.server.optimizing.OptimizingStatus; +import org.apache.amoro.server.optimizing.plan.OptimizingEvaluator; import org.apache.amoro.server.table.TableRuntime; import org.apache.amoro.server.table.TableService; import org.apache.amoro.shade.guava32.com.google.common.base.Function; @@ -149,6 +150,10 @@ public void getTableDetail(Context ctx) { if (serverTableIdentifier.isPresent()) { TableRuntime tableRuntime = tableService.getRuntime(serverTableIdentifier.get()); tableSummary.setOptimizingStatus(tableRuntime.getOptimizingStatus().name()); + OptimizingEvaluator.PendingInput tableRuntimeSummary = tableRuntime.getTableSummary(); + if (tableRuntimeSummary != null) { + tableSummary.setHealthScore(tableRuntimeSummary.getHealthScore()); + } } else { tableSummary.setOptimizingStatus(OptimizingStatus.IDLE.name()); } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/AbstractPartitionPlan.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/AbstractPartitionPlan.java index 5a87cd9e27..4551decd77 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/AbstractPartitionPlan.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/AbstractPartitionPlan.java @@ -181,6 +181,11 @@ protected interface TaskSplitter { List splitTasks(int targetTaskCount); } + @Override + public int getHealthScore() { + return evaluator.getHealthScore(); + } + @Override public int getFragmentFileCount() { return evaluator().getFragmentFileCount(); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java index ffdb2660ce..013f11f87a 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java @@ -369,6 +369,59 @@ public boolean anyDeleteExist() { return equalityDeleteFileCount > 0 || posDeleteFileCount > 0; } + @Override + public int getHealthScore() { + long dataFilesSize = getFragmentFileSize() + getSegmentFileSize(); + long dataFiles = getFragmentFileCount() + getSegmentFileCount(); + long dataRecords = getFragmentFileRecords() + getSegmentFileRecords(); + + double averageDataFileSize = getNormalizedRatio(dataFilesSize, dataFiles); + double eqDeleteRatio = getNormalizedRatio(equalityDeleteFileRecords, dataRecords); + double posDeleteRatio = getNormalizedRatio(posDeleteFileRecords, dataRecords); + + double tablePenaltyFactor = getTablePenaltyFactor(dataFiles, dataFilesSize); + return (int) + Math.ceil( + 100 + - tablePenaltyFactor + * (40 * getSmallFilePenaltyFactor(averageDataFileSize) + + 40 * getEqDeletePenaltyFactor(eqDeleteRatio) + + 20 * getPosDeletePenaltyFactor(posDeleteRatio))); + } + + private double getEqDeletePenaltyFactor(double eqDeleteRatio) { + double eqDeleteRatioThreshold = config.getMajorDuplicateRatio(); + return getNormalizedRatio(eqDeleteRatio, eqDeleteRatioThreshold); + } + + private double getPosDeletePenaltyFactor(double posDeleteRatio) { + double posDeleteRatioThreshold = config.getMajorDuplicateRatio() * 2; + return getNormalizedRatio(posDeleteRatio, posDeleteRatioThreshold); + } + + private double getSmallFilePenaltyFactor(double averageDataFileSize) { + return 1 - getNormalizedRatio(averageDataFileSize, minTargetSize); + } + + private double getTablePenaltyFactor(long dataFiles, long dataFilesSize) { + // if the number of table files is less than or equal to 1, + // there is no penalty, i.e., the table is considered to be perfectly healthy + if (dataFiles <= 1) { + return 0; + } + // The small table has very little impact on performance, + // so there is only a small penalty + return getNormalizedRatio(dataFiles, config.getMinorLeastFileCount()) + * getNormalizedRatio(dataFilesSize, config.getTargetSize()); + } + + private double getNormalizedRatio(double numerator, double denominator) { + if (denominator <= 0) { + return 0; + } + return Math.min(numerator, denominator) / denominator; + } + @Override public int getFragmentFileCount() { return fragmentFileCount; diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java index 7ce4c3e2dd..765bdf3bb1 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java @@ -200,10 +200,12 @@ public static class PendingInput { private long equalityDeleteBytes = 0L; private long equalityDeleteFileRecords = 0L; private long positionalDeleteFileRecords = 0L; + private int healthScore = -1; // -1 means not calculated public PendingInput() {} public PendingInput(Collection evaluators) { + double totalHealthScore = 0; for (PartitionEvaluator evaluator : evaluators) { partitions .computeIfAbsent(evaluator.getPartition().first(), ignore -> Sets.newHashSet()) @@ -217,7 +219,9 @@ public PendingInput(Collection evaluators) { equalityDeleteBytes += evaluator.getEqualityDeleteFileSize(); equalityDeleteFileRecords += evaluator.getEqualityDeleteFileRecords(); equalityDeleteFileCount += evaluator.getEqualityDeleteFileCount(); + totalHealthScore += evaluator.getHealthScore(); } + healthScore = (int) Math.ceil(totalHealthScore / evaluators.size()); } public Map> getPartitions() { @@ -260,6 +264,10 @@ public long getPositionalDeleteFileRecords() { return positionalDeleteFileRecords; } + public int getHealthScore() { + return healthScore; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -273,6 +281,7 @@ public String toString() { .add("equalityDeleteBytes", equalityDeleteBytes) .add("equalityDeleteFileRecords", equalityDeleteFileRecords) .add("positionalDeleteFileRecords", positionalDeleteFileRecords) + .add("healthScore", healthScore) .toString(); } } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/PartitionEvaluator.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/PartitionEvaluator.java index 974703bf07..cc99b2c28b 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/PartitionEvaluator.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/PartitionEvaluator.java @@ -81,6 +81,9 @@ interface Weight extends Comparable {} */ OptimizingType getOptimizingType(); + /** Get health score of this partition. */ + int getHealthScore(); + /** Get the count of fragment files involved in optimizing. */ int getFragmentFileCount(); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableRuntime.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableRuntime.java index 9884d2e785..73855d3a0f 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableRuntime.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableRuntime.java @@ -392,6 +392,10 @@ public OptimizingEvaluator.PendingInput getPendingInput() { return pendingInput; } + public OptimizingEvaluator.PendingInput getTableSummary() { + return tableSummary; + } + private boolean updateConfigInternal(Map properties) { TableConfiguration newTableConfig = TableConfigurations.parseTableConfig(properties); if (tableConfiguration.equals(newTableConfig)) { diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java index 5cd16e26fb..7d4dfd8601 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableSummaryMetrics.java @@ -112,13 +112,20 @@ public class TableSummaryMetrics { .withTags("catalog", "database", "table") .build(); - // table summary snapshots number metrics + // table summary snapshots number metric public static final MetricDefine TABLE_SUMMARY_SNAPSHOTS = defineGauge("table_summary_snapshots") .withDescription("Number of snapshots in the table") .withTags("catalog", "database", "table") .build(); + // table summary health score metric + public static final MetricDefine TABLE_SUMMARY_HEALTH_SCORE = + defineGauge("table_summary_health_score") + .withDescription("Health score of the table") + .withTags("catalog", "database", "table") + .build(); + private final ServerTableIdentifier identifier; private final List registeredMetricKeys = Lists.newArrayList(); private MetricRegistry globalRegistry; @@ -136,6 +143,7 @@ public class TableSummaryMetrics { private long dataFilesRecords = 0L; private long equalityDeleteFilesRecords = 0L; private long snapshots = 0L; + private long healthScore = 0L; public TableSummaryMetrics(ServerTableIdentifier identifier) { this.identifier = identifier; @@ -191,9 +199,12 @@ public void register(MetricRegistry registry) { TABLE_SUMMARY_EQUALITY_DELETE_FILES_RECORDS, (Gauge) () -> equalityDeleteFilesRecords); - // register snapshots number metrics + // register snapshots number metric registerMetric(registry, TABLE_SUMMARY_SNAPSHOTS, (Gauge) () -> snapshots); + // register health score metric + registerMetric(registry, TABLE_SUMMARY_HEALTH_SCORE, (Gauge) () -> healthScore); + globalRegistry = registry; } } @@ -231,6 +242,8 @@ public void refresh(OptimizingEvaluator.PendingInput tableSummary) { positionDeleteFilesRecords = tableSummary.getPositionalDeleteFileRecords(); dataFilesRecords = tableSummary.getDataFileRecords(); equalityDeleteFilesRecords = tableSummary.getEqualityDeleteFileRecords(); + + healthScore = tableSummary.getHealthScore(); } public void refreshSnapshots(MixedTable table) { diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java index 1b4632b29c..e0b992e784 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java @@ -24,6 +24,7 @@ import static org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_EQUALITY_DELETE_FILES; import static org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_EQUALITY_DELETE_FILES_RECORDS; import static org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_EQUALITY_DELETE_FILES_SIZE; +import static org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_HEALTH_SCORE; import static org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_POSITION_DELETE_FILES; import static org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_POSITION_DELETE_FILES_RECORDS; import static org.apache.amoro.server.table.TableSummaryMetrics.TABLE_SUMMARY_POSITION_DELETE_FILES_SIZE; @@ -170,6 +171,8 @@ public void testTableSummaryMetrics() { Gauge snapshots = getMetric(metrics, identifier, TABLE_SUMMARY_SNAPSHOTS); + Gauge healthScore = getMetric(metrics, identifier, TABLE_SUMMARY_HEALTH_SCORE); + Assertions.assertEquals(0, totalFiles.getValue()); Assertions.assertEquals(0, dataFiles.getValue()); Assertions.assertEquals(0, posDelFiles.getValue()); @@ -184,6 +187,7 @@ public void testTableSummaryMetrics() { Assertions.assertEquals(0, dataRecords.getValue()); Assertions.assertEquals(0, posDelRecords.getValue()); Assertions.assertEquals(0, eqDelRecords.getValue()); + Assertions.assertEquals(0, healthScore.getValue()); // refresh metrics initTableWithFiles(); @@ -202,6 +206,7 @@ public void testTableSummaryMetrics() { Assertions.assertTrue(posDelRecords.getValue() > 0); Assertions.assertTrue(snapshots.getValue() > 0); + Assertions.assertTrue(healthScore.getValue() > 0); } private Gauge getMetric( diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/TableSummary.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/TableSummary.java index 95e611e7fa..641f124e2c 100644 --- a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/TableSummary.java +++ b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/TableSummary.java @@ -26,6 +26,7 @@ public class TableSummary { private String tableFormat; private long records; private String optimizingStatus; + private int healthScore = -1; // -1 means not calculated public TableSummary() {} @@ -71,4 +72,13 @@ public String getOptimizingStatus() { public void setOptimizingStatus(String optimizingStatus) { this.optimizingStatus = optimizingStatus; } + + /** Current table health score */ + public int getHealthScore() { + return healthScore; + } + + public void setHealthScore(int healthScore) { + this.healthScore = healthScore; + } } diff --git a/amoro-web/mock/modules/table.js b/amoro-web/mock/modules/table.js index 0e94541caf..a3e61a14d4 100644 --- a/amoro-web/mock/modules/table.js +++ b/amoro-web/mock/modules/table.js @@ -76,7 +76,10 @@ export default [ "file": 2, "size": "1.79KB", "tableFormat": "Iceberg(V1)", - "averageFile": "918.00B" + "averageFile": "918.00B", + "records":24, + "optimizingStatus":"IDLE", + "healthScore":100, }, "baseLocation": "/mnt/dfs/4/warehouse_public/db/user", "filter": null, diff --git a/amoro-web/src/language/en.ts b/amoro-web/src/language/en.ts index 60d494f0dd..06ba5513ef 100644 --- a/amoro-web/src/language/en.ts +++ b/amoro-web/src/language/en.ts @@ -17,6 +17,7 @@ */ export default { + healthScore: 'Health Score', overview: 'Overview', catalogs: 'Catalogs', catalog: 'Catalog', diff --git a/amoro-web/src/language/zh.ts b/amoro-web/src/language/zh.ts index 6bafedd9e6..0936135cc3 100644 --- a/amoro-web/src/language/zh.ts +++ b/amoro-web/src/language/zh.ts @@ -17,6 +17,7 @@ */ export default { + healthScore: '健康度', overview: '总览', catalogs: '目录', catalog: '目录', diff --git a/amoro-web/src/types/common.type.ts b/amoro-web/src/types/common.type.ts index 10b26d8879..4af04c14dc 100644 --- a/amoro-web/src/types/common.type.ts +++ b/amoro-web/src/types/common.type.ts @@ -60,6 +60,7 @@ export interface IKeyAndValue { } export interface IBaseDetailInfo { optimizingStatus: string + records: string tableType: string tableName: string createTime: string @@ -68,6 +69,7 @@ export interface IBaseDetailInfo { averageFile: string tableFormat: string hasPartition: boolean + healthScore: number } export interface DetailColumnItem { diff --git a/amoro-web/src/views/tables/index.vue b/amoro-web/src/views/tables/index.vue index 6a911e908e..18fa8fca73 100644 --- a/amoro-web/src/views/tables/index.vue +++ b/amoro-web/src/views/tables/index.vue @@ -54,11 +54,13 @@ export default defineComponent({ isSecondaryNav: false, baseInfo: { optimizingStatus: '', + records: '', tableType: '', tableName: '', createTime: '', tableFormat: '', hasPartition: false, + healthScore: -1, } as IBaseDetailInfo, detailLoaded: false, }) @@ -150,6 +152,10 @@ export default defineComponent({

{{ $t('tableFormat') }}: {{ baseInfo.tableFormat }}

+ +

+ {{ $t('healthScore') }}: {{ baseInfo.healthScore == null || baseInfo.healthScore < 0 ? 'N/A' : baseInfo.healthScore }} +

@@ -178,9 +184,11 @@ export default defineComponent({ border: 1px solid #e8e8f0; padding: 12px 0; min-height: 100%; + .create-time { margin-top: 12px; } + .tables-menu-wrap { position: fixed; width: 100%; @@ -189,6 +197,7 @@ export default defineComponent({ left: 200px; z-index: 100; } + .table-name { font-size: 24px; line-height: 1.5; @@ -196,16 +205,20 @@ export default defineComponent({ max-width: 100%; padding-left: 24px; } + .table-info { padding: 12px 24px 0 24px; + .text-color { color: #7CB305; } } + .table-edit { font-size: 18px; padding-right: 12px; } + :deep(.ant-tabs-nav) { padding-left: 12px; margin-bottom: 0; diff --git a/docs/user-guides/metrics.md b/docs/user-guides/metrics.md index cc86bcb0d9..d65ce4c9ed 100644 --- a/docs/user-guides/metrics.md +++ b/docs/user-guides/metrics.md @@ -100,18 +100,19 @@ Amoro has supported built-in metrics to measure status of table self-optimizing ## table summary metrics -| Metric Name | Type | Tags | Description | -|---------------------------------------------|---------|--------------------------|-----------------------------------------------| -| table_summary_total_files | Gauge | catalog, database, table | Total number of files in the table | -| table_summary_data_files | Gauge | catalog, database, table | Number of data files in the table | -| table_summary_equality_delete_files | Gauge | catalog, database, table | Number of equality delete files in the table | -| table_summary_position_delete_files | Gauge | catalog, database, table | Number of position delete files in the table | -| table_summary_total_files_size | Gauge | catalog, database, table | Total size of files in the table | -| table_summary_data_files_size | Gauge | catalog, database, table | Size of data files in the table | -| table_summary_equality_delete_files_size | Gauge | catalog, database, table | Size of equality delete files in the table | -| table_summary_position_delete_files_size | Gauge | catalog, database, table | Size of position delete files in the table | -| table_summary_total_records | Gauge | catalog, database, table | Total records in the table | -| table_summary_data_files_records | Gauge | catalog, database, table | Records of data files in the table | -| table_summary_equality_delete_files_records | Gauge | catalog, database, table | Records of equality delete files in the table | -| table_summary_position_delete_files_records | Gauge | catalog, database, table | Records of position delete files in the table | -| table_summary_snapshots | Gauge | catalog, database, table | Number of snapshots in the table | \ No newline at end of file +| Metric Name | Type | Tags | Description | +|-----------------------------------------------|---------|--------------------------|-----------------------------------------------| +| table_summary_total_files | Gauge | catalog, database, table | Total number of files in the table | +| table_summary_data_files | Gauge | catalog, database, table | Number of data files in the table | +| table_summary_equality_delete_files | Gauge | catalog, database, table | Number of equality delete files in the table | +| table_summary_position_delete_files | Gauge | catalog, database, table | Number of position delete files in the table | +| table_summary_total_files_size | Gauge | catalog, database, table | Total size of files in the table | +| table_summary_data_files_size | Gauge | catalog, database, table | Size of data files in the table | +| table_summary_equality_delete_files_size | Gauge | catalog, database, table | Size of equality delete files in the table | +| table_summary_position_delete_files_size | Gauge | catalog, database, table | Size of position delete files in the table | +| table_summary_total_records | Gauge | catalog, database, table | Total records in the table | +| table_summary_data_files_records | Gauge | catalog, database, table | Records of data files in the table | +| table_summary_equality_delete_files_records | Gauge | catalog, database, table | Records of equality delete files in the table | +| table_summary_position_delete_files_records | Gauge | catalog, database, table | Records of position delete files in the table | +| table_summary_snapshots | Gauge | catalog, database, table | Number of snapshots in the table | +| table_summary_health_score | Gauge | catalog, database, table | Health score of the table | \ No newline at end of file