From eed0a4687b568d6255b79d5f90c9e7f179aee0d4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 13 Feb 2025 09:36:08 +0800 Subject: [PATCH] Added metrics to DeviceSchemaCache's memory and entries num (#14826) * Partial * partial * put * partial --- .../dualkeycache/IDualKeyCacheStats.java | 6 ++++ .../schema/dualkeycache/impl/CacheStats.java | 24 +++++++++++-- .../dualkeycache/impl/DualKeyCacheImpl.java | 9 ++++- .../fetcher/cache/TableDeviceSchemaCache.java | 12 +++++++ .../cache/TableDeviceSchemaCacheMetrics.java | 34 +++++++++++++++++++ .../commons/service/metric/enums/Metric.java | 1 + 6 files changed, 83 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java index 9ed842cc602d..9247e20a9c48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java @@ -39,4 +39,10 @@ public interface IDualKeyCacheStats { /** Return current memory usage of dual key cache. */ long memoryUsage(); + + /** Return capacity of dual key cache. */ + long capacity(); + + /** Return entries num of dual key cache */ + long entriesCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java index 817092d3eb90..99c331962a20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java @@ -31,6 +31,7 @@ class CacheStats implements IDualKeyCacheStats { private final long memoryThreshold; private final AtomicLong memoryUsage = new AtomicLong(0); + private final AtomicLong entriesCount = new AtomicLong(0); private final AtomicLong requestCount = new AtomicLong(0); private final AtomicLong hitCount = new AtomicLong(0); @@ -68,6 +69,14 @@ void recordMiss(int num) { requestCount.getAndAdd(num); } + void increaseEntryCount() { + entriesCount.incrementAndGet(); + } + + void decreaseEntryCount() { + entriesCount.decrementAndGet(); + } + @Override public long requestCount() { return requestCount.get(); @@ -96,13 +105,24 @@ public long memoryUsage() { return memoryUsage.get(); } + @Override + public long capacity() { + return memoryThreshold; + } + + @Override + public long entriesCount() { + return entriesCount.get(); + } + void reset() { - resetMemoryUsage(); + resetMemoryUsageAndEntriesCount(); hitCount.set(0); requestCount.set(0); } - void resetMemoryUsage() { + void resetMemoryUsageAndEntriesCount() { memoryUsage.set(0); + entriesCount.set(0); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java index 126f5314495d..7a4c816f4078 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java @@ -160,6 +160,7 @@ public void put(final FK firstKey, final SK secondKey, final V value) { cacheEntry = cacheEntryManager.createCacheEntry(secondKey, value, finalCacheEntryGroup); cacheEntryManager.put(cacheEntry); + cacheStats.increaseEntryCount(); usedMemorySize.getAndAdd(sizeComputer.computeSecondKeySize(sk)); } else { final V existingValue = cacheEntry.getValue(); @@ -208,6 +209,7 @@ public void update( cacheEntryManager.createCacheEntry( secondKey, value, finalCacheEntryGroup); cacheEntryManager.put(entry); + cacheStats.increaseEntryCount(); usedMemorySize.getAndAdd( sizeComputer.computeSecondKeySize(sk) + sizeComputer.computeValueSize(entry.getValue())); @@ -304,6 +306,7 @@ private int evictOneCacheEntry() { final ICacheEntryGroup belongedGroup = evictCacheEntry.getBelongedGroup(); evictCacheEntry.setBelongedGroup(null); belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); + cacheStats.decreaseEntryCount(); evictedSize.getAndAdd(sizeComputer.computeSecondKeySize(evictCacheEntry.getSecondKey())); if (belongedGroup.isEmpty()) { @@ -334,7 +337,7 @@ public void invalidateAll() { private void executeInvalidateAll() { firstKeyMap.clear(); cacheEntryManager.cleanUp(); - cacheStats.resetMemoryUsage(); + cacheStats.resetMemoryUsageAndEntriesCount(); } @Override @@ -364,6 +367,7 @@ public void invalidate(final FK firstKey) { it.hasNext(); ) { final Map.Entry entry = it.next(); if (cacheEntryManager.invalidate(entry.getValue())) { + cacheStats.decreaseEntryCount(); estimateSize += sizeComputer.computeSecondKeySize(entry.getKey()) + sizeComputer.computeValueSize(entry.getValue().getValue()); @@ -387,6 +391,7 @@ public void invalidate(final FK firstKey, final SK secondKey) { final T entry = cacheEntryGroup.getCacheEntry(secondKey); if (Objects.nonNull(entry) && cacheEntryManager.invalidate(entry)) { + cacheStats.decreaseEntryCount(); usedMemorySize.getAndAdd( sizeComputer.computeSecondKeySize(entry.getSecondKey()) + sizeComputer.computeValueSize(entry.getValue())); @@ -418,6 +423,7 @@ public void invalidate(final FK firstKey, final Predicate secondKeyChecker) it.hasNext(); ) { final Map.Entry entry = it.next(); if (cacheEntryManager.invalidate(entry.getValue())) { + cacheStats.decreaseEntryCount(); cacheEntryGroup.removeCacheEntry(entry.getKey()); estimateSize.addAndGet( sizeComputer.computeSecondKeySize(entry.getKey()) @@ -452,6 +458,7 @@ public void invalidate( } if (cacheEntryManager.invalidate(entry.getValue())) { + cacheStats.decreaseEntryCount(); entryGroup.removeCacheEntry(entry.getKey()); estimateSize.addAndGet( sizeComputer.computeSecondKeySize(entry.getKey()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index 195e6e896ced..9a0af25d1cf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -508,6 +508,18 @@ long getRequestCount() { return dualKeyCache.stats().requestCount(); } + long getMemoryUsage() { + return dualKeyCache.stats().memoryUsage(); + } + + long capacity() { + return dualKeyCache.stats().capacity(); + } + + long entriesCount() { + return dualKeyCache.stats().entriesCount(); + } + void invalidateLastCache(final @Nonnull String database) { readWriteLock.writeLock().lock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java index 56e1b23de456..e65c418c0c1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java @@ -29,6 +29,10 @@ import java.util.Objects; public class TableDeviceSchemaCacheMetrics implements IMetricSet { + + private static final String SCHEMA_CACHE_TOTAL_USAGE = "schema_cache_total_usage"; + private static final String SCHEMA_CACHE_MEM_CAPACITY = "schema_cache_mem_capacity"; + private final TableDeviceSchemaCache tableDeviceSchemaCache; public TableDeviceSchemaCacheMetrics(final TableDeviceSchemaCache dataNodeSchemaCache) { @@ -55,6 +59,25 @@ public void bindTo(final AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + tableDeviceSchemaCache, + TableDeviceSchemaCache::getMemoryUsage, + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + tableDeviceSchemaCache, + TableDeviceSchemaCache::capacity, + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.createAutoGauge( + Metric.CACHE_ENTRIES_NUM.toString(), + MetricLevel.IMPORTANT, + tableDeviceSchemaCache, + TableDeviceSchemaCache::entriesCount); } @Override @@ -73,6 +96,17 @@ public void unbindFrom(final AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.remove(MetricType.AUTO_GAUGE, Metric.CACHE_ENTRIES_NUM.toString()); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 906c9f05968e..65ecf6280b55 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -98,6 +98,7 @@ public enum Metric { MEM("mem"), CACHE("cache"), CACHE_HIT_RATE("cache_hit"), + CACHE_ENTRIES_NUM("cache_entries_num"), QUANTITY("quantity"), LEADER_QUANTITY("leader_quantity"), SCHEMA_REGION("schema_region"),