From 511ed6c94b60217df6bf28e1590245fb2dc633fb Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 27 Sep 2024 13:06:44 -0700 Subject: [PATCH] These tests were attempting to test the ability of the MemoryRequestServer to correctly move a direct sketch onto the heap. But the implementations of the tests were not correct. --- .../datasketches/hll/DirectAuxHashMapTest.java | 13 +++++++++---- .../quantiles/DirectQuantilesMemoryRequestTest.java | 12 ++++++++++++ .../datasketches/quantiles/DoublesSketchTest.java | 6 ++++++ .../theta/DirectQuickSelectSketchTest.java | 6 ++++++ .../apache/datasketches/theta/UnionImplTest.java | 3 +++ 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 71774783c..c1ef688ca 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -43,13 +43,16 @@ public class DirectAuxHashMapTest { @Test - public void checkGrow() { + public void checkGrow() { //It is very rare, but this forces an HLL_4 to exceed its computed memory size. int lgConfigK = 4; TgtHllType tgtHllType = TgtHllType.HLL_4; int n = 8; //put lgConfigK == 4 into HLL mode int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); HllSketch hllSketch; WritableMemory wmem = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmemCopy = wmem; //copy of wmem + assertTrue(wmemCopy.isDirect()); //original copy of wmem is off-heap + assertTrue(wmemCopy.isAlive()); //original copy of wmem is Alive hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); for (int i = 0; i < n; i++) { hllSketch.update(i); @@ -57,7 +60,7 @@ public void checkGrow() { hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values hllSketch.couponUpdate(HllUtil.pair(8, 15)); hllSketch.couponUpdate(HllUtil.pair(9, 15)); - //println(hllSketch.toString(true, true, true, true)); + //println(hllSketch.toString(true, true, true, true)); // DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); assertTrue(hllSketch.isMemory()); @@ -75,19 +78,21 @@ public void checkGrow() { byteArray = hllSketch.toUpdatableByteArray(); WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); hllSketch2 = HllSketch.writableWrap(wmem2); - //println(hllSketch2.toString(true, true, true, true)); + //println(hllSketch2.toString(true, true, true, true)); // DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); //Check grow to on-heap hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow - //println(hllSketch.toString(true, true, true, true)); + //println(hllSketch.toString(true, true, true, true)); // dha = (DirectHllArray) hllSketch.hllSketchImpl; assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); assertEquals(dha.getAuxHashMap().getAuxCount(), 4); assertTrue(hllSketch.isMemory()); assertFalse(hllSketch.isOffHeap()); + assertTrue(wmemCopy.isDirect()); //original copy of wmem was off-heap and still is + assertFalse(wmemCopy.isAlive()); //original copy of wmem has been closed } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 2ec2fbe27..cfa14b60c 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -48,6 +48,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application //########## Owning Implementation // This part would actually be part of the Memory owning implementation so it is faked here WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmemCopy = wmem; println("Initial mem size: " + wmem.getCapacity()); //########## Receiving Application @@ -70,6 +71,8 @@ public void checkLimitedMemoryScenarios() { //Requesting application // so the the wmem reference is invalid. Use the sketch to get the last memory reference. WritableMemory lastMem = usk1.getMemory(); println("Final mem size: " + usk1.getMemory().getCapacity()); + assertTrue(wmemCopy.isDirect()); + assertFalse(wmemCopy.isAlive()); } @Test @@ -79,6 +82,7 @@ public void checkGrowBaseBuf() { final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmemCopy = wmem; println("Initial mem size: " + wmem.getCapacity()); final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { @@ -88,6 +92,8 @@ public void checkGrowBaseBuf() { println("curCombBufItemCap: " + currentSpace); assertEquals(currentSpace, 2 * k); println("last Mem Cap: " + usk1.getMemory().getCapacity()); + assertTrue(wmemCopy.isDirect()); + assertFalse(wmemCopy.isAlive()); } @Test @@ -97,6 +103,7 @@ public void checkGrowCombBuf() { final int initBytes = ((2 * k) + 4) << 3; //just room for BB WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmemCopy = wmem; println("Initial mem size: " + wmem.getCapacity()); final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { @@ -108,6 +115,8 @@ public void checkGrowCombBuf() { final int newSpace = usk1.getCombinedBufferItemCapacity(); println("newCombBurItemCap: " + newSpace); assertEquals(newCB.length, 3 * k); + assertTrue(wmemCopy.isDirect()); + assertFalse(wmemCopy.isAlive()); } @Test @@ -119,6 +128,7 @@ public void checkGrowFromWrappedEmptySketch() { final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmemCopy = wmem; origSketchMem.copyTo(0, wmem, 0, initBytes); UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); assertTrue(wmem.isSameResource(usk2.getMemory())); @@ -135,6 +145,8 @@ public void checkGrowFromWrappedEmptySketch() { final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); assertEquals(mem2.getCapacity(), expectedSize); + assertTrue(wmemCopy.isDirect()); + assertFalse(wmemCopy.isAlive()); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index cf8e142b7..fd1b7a72a 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -140,18 +140,22 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { WritableMemory wmem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmemCopy = wmem; UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); Assert.assertTrue(sketch.isSameResource(wmem)); for (int i = 0; i < 1000; i++) { sketch.update(i); } println(sketch.toString()); + assertTrue(wmemCopy.isDirect()); + assertFalse(wmemCopy.isAlive()); } @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; WritableMemory wmem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer()); + WritableMemory wmemCopy = wmem; UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); Assert.assertTrue(sketch.isSameResource(wmem)); for (; i < 1000; i++) { @@ -163,6 +167,8 @@ public void directSketchShouldMoveOntoHeapEventually2() { } } assertFalse(wmem.isAlive()); + assertTrue(wmemCopy.isDirect()); + assertFalse(wmemCopy.isAlive()); } @Test diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 9b129daaf..2aa195a0c 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -781,13 +781,16 @@ public void checkMoveAndResize() { int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory + WritableMemory wmemCopy = wmem; UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); assertTrue(sketch.isSameResource(wmem)); for (int i = 0; i < u; i++) { sketch.update(i); } Memory mem = sketch.getMemory(); assertTrue(mem.isAlive()); assertFalse(mem.isDirect()); //now on heap. + assertTrue(wmemCopy.isDirect()); //original copy assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer + assertFalse(wmemCopy.isAlive()); //original copy closed } @Test @@ -796,6 +799,7 @@ public void checkReadOnlyRebuildResize() { int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory + WritableMemory wmemCopy = wmem; UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); for (int i = 0; i < u; i++) { sketch.update(i); } double est1 = sketch.getEstimate(); @@ -808,6 +812,8 @@ public void checkReadOnlyRebuildResize() { assertTrue(mem2.isAlive()); assertFalse(mem2.isDirect()); //now on heap assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer + assertTrue(wmemCopy.isDirect()); + assertFalse(wmemCopy.isAlive()); try { roSketch.rebuild(); fail(); diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index 2fbd7bdcf..4bade62f7 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -193,6 +193,7 @@ public void checkMoveAndResizeOffHeap() { final int bytes = Sketches.getMaxUpdateSketchBytes(k); WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2); //not really used, except as a reference. WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2); //too small, forces new allocation on heap + WritableMemory wmem2Copy = wmem2; final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); assertTrue(sketch.isSameResource(wmem)); //also testing the isSameResource function @@ -206,6 +207,8 @@ public void checkMoveAndResizeOffHeap() { assertFalse(union2.isSameResource(wmem2)); //obviously not wmem.close(); //empty, but we must close it anyway. assertFalse(wmem2.isAlive());//previously closed via the DefaultMemoryRequestServer. + assertTrue(wmem2Copy.isDirect()); + assertFalse(wmem2Copy.isAlive()); } @Test