From ee7dfd80885fa321918db2da184e9398b105321f Mon Sep 17 00:00:00 2001
From: Ralf Schmelter <ralf.schmelter@sap.com>
Date: Thu, 5 Sep 2024 16:19:31 +0200
Subject: [PATCH] Make test for unique stacks more rubust

---
 .../runtime/malloctrace/MallocHooksTest.java  | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/test/hotspot/jtreg/runtime/malloctrace/MallocHooksTest.java b/test/hotspot/jtreg/runtime/malloctrace/MallocHooksTest.java
index 13a84996996..b41e81408eb 100644
--- a/test/hotspot/jtreg/runtime/malloctrace/MallocHooksTest.java
+++ b/test/hotspot/jtreg/runtime/malloctrace/MallocHooksTest.java
@@ -35,7 +35,9 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStreamReader;
+import java.io.IOException;
 import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -595,11 +597,25 @@ private static void testUniqueStacks() throws Exception {
                                   "-XX:MallocTraceStackDepth=14").start();
         checkIsAttachable(p);
         p.getInputStream().read();
-        OutputAnalyzer oa = callJcmd(p, "MallocTrace.dump", "-percentage=100");
+        // The output can be large, so dump it directly, since the output of
+        // jcmd is temporary hold in the JVM and there is a limit of 100 MB.
+        final String[] stacks = new String[1];
+        Thread t = new Thread(new Runnable() {
+            public void run() {
+                try {
+                    stacks[0] = new String(p.getErrorStream().readAllBytes(), StandardCharsets.UTF_8);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+        t.start();
+        OutputAnalyzer oa = callJcmd(p, "MallocTrace.dump", "-percentage=100", "-dump-file=stderr");
         oa.shouldHaveExitValue(0);
         p.destroy();
+        t.join();
 
-        MallocTraceResult result = MallocTraceResult.fromString(oa.getOutput());
+        MallocTraceResult result = MallocTraceResult.fromString(stacks[0]);
         HashSet<Stack> seenStacks = new HashSet<>();
 
         for (int i = 0; i < result.nrOfStacks(); ++i) {