diff --git a/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java index 16a3dbb5..4f1d4e3b 100644 --- a/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java @@ -1,5 +1,5 @@ /* - * (C) ActiveViam 2018-2021 + * (C) ActiveViam 2018-2023 * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY * property of ActiveViam. Any unauthorized use, * reproduction or transfer of this material is strictly prohibited @@ -29,11 +29,11 @@ import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.qfs.agg.impl.SingleValueFunction; -import com.qfs.literal.ILiteralType; import com.qfs.multiversion.IEpoch; import com.qfs.pivot.util.impl.MdxNamingUtil; import com.qfs.server.cfg.IActivePivotManagerDescriptionConfig; import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; +import com.qfs.store.Types; import com.quartetfs.biz.pivot.context.impl.QueriesResultLimit; import com.quartetfs.biz.pivot.context.impl.QueriesTimeLimit; import com.quartetfs.biz.pivot.cube.dimension.IDimension; @@ -231,7 +231,7 @@ public class ManagerDescriptionConfig implements IActivePivotManagerDescriptionC /** The name of the folder for measures related to chunks. */ public static final String CHUNK_FOLDER = "Chunk"; /** The name of the folder for measures related to datastore-related chunks. */ - public static final String STORE_CHUNK_FOLDER = "DataStore Chunk"; + public static final String STORE_CHUNK_FOLDER = "Datastore Chunk"; /** The name of the folder for measures related to chunk memory usage. */ public static final String CHUNK_MEMORY_FOLDER = "Chunk Memory"; /** The name of the folder for measures related to vectors. */ @@ -631,6 +631,7 @@ private void chunkMeasures(final ICopperContext context) { .withDescription("the number of freed rows within the chunks") .publish(context) .divide(chunkSize) + .withType(Types.TYPE_DOUBLE) .withFormatter(PERCENT_FORMATTER) .as(DELETED_ROWS_RATIO) .withinFolder(CHUNK_FOLDER) @@ -638,8 +639,8 @@ private void chunkMeasures(final ICopperContext context) { .publish(context); nonWrittenRowsCount - .mapToDouble(a -> a.readDouble(0)) .divide(chunkSize) + .withType(Types.TYPE_DOUBLE) .withFormatter(PERCENT_FORMATTER) .as(NON_WRITTEN_ROWS_RATIO) .withinFolder(CHUNK_FOLDER) @@ -655,7 +656,8 @@ private void chunkMeasures(final ICopperContext context) { .publish(context); Copper.measure(COMMITTED_ROWS_COUNT) - .divide(Copper.measure(CHUNK_SIZE_SUM)) + .divide(chunkSize) + .withType(Types.TYPE_DOUBLE) .withFormatter(PERCENT_FORMATTER) .as(COMMITTED_ROWS_RATIO) .withinFolder(CHUNK_FOLDER) @@ -671,10 +673,9 @@ private void chunkMeasures(final ICopperContext context) { .withDescription("the off-heap size of the chunks") .publish(context); - Copper.measure(COMMITTED_ROWS_COUNT) - .divide(Copper.measure(CHUNK_SIZE_SUM)) - .multiply(Copper.measure(DIRECT_MEMORY_SUM)) - .withType(ILiteralType.LONG) + Copper.measure(COMMITTED_ROWS_RATIO) + .multiply(directMemory) + .mapToLong(a -> (long) a.readDouble(0)) .withFormatter(ByteFormatter.KEY) .as(COMMITTED_CHUNK_MEMORY) .withinFolder(CHUNK_MEMORY_FOLDER) @@ -683,6 +684,7 @@ private void chunkMeasures(final ICopperContext context) { directMemory .divide(directMemory.grandTotal()) + .withType(Types.TYPE_DOUBLE) .withFormatter(PERCENT_FORMATTER) .as(DIRECT_MEMORY_RATIO) .withinFolder(CHUNK_MEMORY_FOLDER) @@ -693,6 +695,7 @@ private void chunkMeasures(final ICopperContext context) { directMemory .divide(Copper.measure(USED_DIRECT)) + .withType(Types.TYPE_DOUBLE) .withFormatter(PERCENT_FORMATTER) .as(USED_MEMORY_RATIO) .withinFolder(CHUNK_MEMORY_FOLDER) @@ -703,6 +706,7 @@ private void chunkMeasures(final ICopperContext context) { directMemory .divide(Copper.measure(MAX_DIRECT)) + .withType(Types.TYPE_DOUBLE) .withFormatter(PERCENT_FORMATTER) .as(MAX_MEMORY_RATIO) .withinFolder(CHUNK_MEMORY_FOLDER) diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java b/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java new file mode 100644 index 00000000..bce1d922 --- /dev/null +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java @@ -0,0 +1,163 @@ +package com.activeviam.mac.statistic.memory; + +import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; +import com.activeviam.pivot.utils.ApplicationInTests; +import com.qfs.monitoring.statistic.memory.IMemoryStatistic; +import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; +import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; +import com.qfs.store.IDatastore; +import com.quartetfs.biz.pivot.IMultiVersionActivePivot; +import com.quartetfs.biz.pivot.dto.CellSetDTO; +import com.quartetfs.biz.pivot.query.impl.MDXQuery; +import com.quartetfs.fwk.AgentException; +import com.quartetfs.fwk.Registry; +import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; +import com.quartetfs.fwk.query.QueryException; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.IntStream; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestRatioMeasures extends ATestMemoryStatistic { + + public static final int ADDED_DATA_SIZE = 20; + private ApplicationInTests monitoredApp; + private ApplicationInTests monitoringApp; + + @BeforeAll + public static void setupRegistry() { + Registry.setContributionProvider(new ClasspathContributionProvider()); + } + + @BeforeEach + public void setup() throws AgentException { + this.monitoredApp = createMicroApplicationWithIsolatedStoreAndKeepAllEpochPolicy(); + + this.monitoredApp + .getDatabase() + .edit( + tm -> + IntStream.range(0, ADDED_DATA_SIZE) + .forEach( + i -> { + tm.add("A", i, i / 10D); + tm.add("B", i, i / 10D); + })); + + // Force to discard all versions + this.monitoredApp.getDatabase().getEpochManager().forceDiscardEpochs(__ -> true); + + // perform GCs before exporting the store data + performGC(); + final MemoryAnalysisService analysisService = + (MemoryAnalysisService) + createService(this.monitoredApp.getDatabase(), this.monitoredApp.getManager()); + final Path exportPath = analysisService.exportMostRecentVersion("testOverview"); + + final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + + // Start a monitoring datastore with the exported data + final ManagerDescriptionConfig config = new ManagerDescriptionConfig(); + final IDatastoreSchemaDescriptionConfig schemaConfig = + new MemoryAnalysisDatastoreDescriptionConfig(); + + this.monitoringApp = + ApplicationInTests.builder() + .withDatastore(schemaConfig.datastoreSchemaDescription()) + .withManager(config.managerDescription()) + .build(); + + resources.register(this.monitoringApp).start(); + + // Fill the monitoring datastore + ATestMemoryStatistic.feedMonitoringApplication( + monitoringApp.getDatabase(), List.of(stats), "storeA"); + + final IMultiVersionActivePivot pivot = + this.monitoringApp + .getManager() + .getActivePivots() + .get(ManagerDescriptionConfig.MONITORING_CUBE); + Assertions.assertThat(pivot).isNotNull(); + } + + @Test + public void testDirectMemoryRatio() throws QueryException { + final IMultiVersionActivePivot pivot = + this.monitoringApp + .getManager() + .getActivePivots() + .get(ManagerDescriptionConfig.MONITORING_CUBE); + + final MDXQuery totalDirectMemory = + new MDXQuery( + "SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS " + + "FROM [MemoryCube]" + + "WHERE [Owners].[Owner].[ALL].[AllMember]"); + + final MDXQuery storeADirectMemory = + new MDXQuery( + "SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS " + + "FROM [MemoryCube]" + + "WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); + + final MDXQuery storeADirectMemoryRatio = + new MDXQuery( + "SELECT NON EMPTY [Measures].[DirectMemory.Ratio] ON COLUMNS " + + "FROM [MemoryCube]" + + "WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); + + final CellSetDTO total = pivot.execute(totalDirectMemory); + final CellSetDTO storeA = pivot.execute(storeADirectMemory); + final CellSetDTO ratio = pivot.execute(storeADirectMemoryRatio); + + Assertions.assertThat(CellSetUtils.extractDoubleValueFromSingleCellDTO(ratio).doubleValue()) + .isEqualTo(0.5D); + Assertions.assertThat(CellSetUtils.extractDoubleValueFromSingleCellDTO(ratio).doubleValue()) + .isEqualTo( + CellSetUtils.extractValueFromSingleCellDTO(storeA) + / CellSetUtils.extractValueFromSingleCellDTO(total).doubleValue()); + } + + @Test + public void testCommittedRowsRatio() throws QueryException { + final IMultiVersionActivePivot pivot = + this.monitoringApp + .getManager() + .getActivePivots() + .get(ManagerDescriptionConfig.MONITORING_CUBE); + + final MDXQuery storeAcommittedRows = + new MDXQuery( + "SELECT NON EMPTY [Measures].[CommittedRows.COUNT] ON COLUMNS " + + "FROM [MemoryCube]" + + "WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); + + final MDXQuery storeAchunkSize = + new MDXQuery( + "SELECT NON EMPTY [Measures].[ChunkSize.SUM] ON COLUMNS " + + "FROM [MemoryCube]" + + "WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); + + final MDXQuery storeAcommittedRowsRatio = + new MDXQuery( + "SELECT NON EMPTY [Measures].[CommittedRows.Ratio] ON COLUMNS " + + "FROM [MemoryCube]" + + "WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); + + final CellSetDTO committedRows = pivot.execute(storeAcommittedRows); + final CellSetDTO chunkSize = pivot.execute(storeAchunkSize); + final CellSetDTO ratio = pivot.execute(storeAcommittedRowsRatio); + + Assertions.assertThat(CellSetUtils.extractDoubleValueFromSingleCellDTO(ratio).doubleValue()) + .isNotIn(0D, 1D); + Assertions.assertThat(CellSetUtils.extractDoubleValueFromSingleCellDTO(ratio).doubleValue()) + .isEqualTo( + CellSetUtils.extractValueFromSingleCellDTO(committedRows) + / CellSetUtils.extractValueFromSingleCellDTO(chunkSize).doubleValue()); + } +}