8380011: Path-to-gcroots search should not trigger stack overflows

src/hotspot/share/jfr/leakprofiler/chains/dfsClosure.cpp

@@ -35,6 +35,7 @@
 #include "memory/resourceArea.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/os.hpp"
 #include "utilities/align.hpp"
 
 UnifiedOopRef DFSClosure::_reference_stack[max_dfs_depth];
@@ -68,9 +69,27 @@ void DFSClosure::find_leaks_from_root_set(EdgeStore* edge_store,
   rs.process();
 }
 
+static address calculate_headroom_limit() {
+  static constexpr size_t required_headroom = K * 64;
+  const Thread* const t = Thread::current_or_null();
+  return t->stack_end() + required_headroom;
+}
+
 DFSClosure::DFSClosure(EdgeStore* edge_store, BitSet* mark_bits, const Edge* start_edge)
   :_edge_store(edge_store), _mark_bits(mark_bits), _start_edge(start_edge),
-  _max_depth(max_dfs_depth), _depth(0), _ignore_root_set(false) {
+  _max_depth(max_dfs_depth), _depth(0), _ignore_root_set(false),
+  _headroom_limit(calculate_headroom_limit()) {
+}
+
+bool DFSClosure::have_headroom() const {
+  const address sp = (address) os::current_stack_pointer();
+#ifdef ASSERT
+  const Thread* const t = Thread::current_or_null();
+  assert(t->is_VM_thread(), "invariant");
+  assert(t->is_in_full_stack(_headroom_limit), "invariant");
+  assert(t->is_in_full_stack(sp), "invariant");
+#endif
+  return sp > _headroom_limit;
 }
 
 void DFSClosure::closure_impl(UnifiedOopRef reference, const oop pointee) {
@@ -98,7 +117,7 @@ void DFSClosure::closure_impl(UnifiedOopRef reference, const oop pointee) {
     }
   }
   assert(_max_depth >= 1, "invariant");
-  if (_depth < _max_depth - 1) {
+  if (_depth < _max_depth - 1 && have_headroom()) {
     _depth++;
     pointee->oop_iterate(this);
     assert(_depth > 0, "invariant");

src/hotspot/share/jfr/leakprofiler/chains/dfsClosure.hpp

@@ -46,12 +46,15 @@ class DFSClosure : public BasicOopIterateClosure {
   size_t _max_depth;
   size_t _depth;
   bool _ignore_root_set;
+  const address _headroom_limit;
 
   DFSClosure(EdgeStore* edge_store, BitSet* mark_bits, const Edge* start_edge);
 
   void add_chain();
   void closure_impl(UnifiedOopRef reference, const oop pointee);
 
+  bool have_headroom() const;
+
  public:
   virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS_EXCEPT_REFERENT; }
 

test/jdk/TEST.groups

@@ -528,6 +528,7 @@ jdk_jfr_sanity = \
     jdk/jfr/event/gc/collection/TestGCWithFasttime.java \
     jdk/jfr/event/gc/configuration/TestGCConfigurationEvent.java \
     jdk/jfr/event/metadata/TestDefaultConfigurations.java \
+    jdk/jfr/event/oldobject/TestDFSWithSmallStack.java \
     jdk/jfr/startupargs/TestDumpOnExit.java \
     jdk/jfr/api/consumer/recordingstream/TestBasics.java
 

test/jdk/jdk/jfr/event/oldobject/OldObjects.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -276,4 +276,16 @@ public static void validateReferenceChainLimit(RecordedEvent e, int maxLength) {
             throw new RuntimeException("Reference chain max length not respected. Found a chain of length " + length);
         }
     }
+
+    public static int countChains(List<RecordedEvent> events) throws IOException {
+        int found = 0;
+        for (RecordedEvent e : events) {
+            RecordedObject ro = e.getValue("object");
+            if (ro.getValue("referrer") != null) {
+                found++;
+            }
+        }
+        System.out.println("Found chains: " + found);
+        return found;
+    }
 }

test/jdk/jdk/jfr/event/oldobject/TestDFSWithSmallStack.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026, IBM Corp.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package jdk.jfr.event.oldobject;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import jdk.jfr.Recording;
+import jdk.jfr.consumer.RecordedEvent;
+import jdk.jfr.internal.test.WhiteBox;
+import jdk.test.lib.jfr.EventNames;
+import jdk.test.lib.jfr.Events;
+
+/**
+ * @test id=dfsonly
+ * @summary Tests that DFS works with a small stack
+ * @library /test/lib /test/jdk
+ * @requires vm.hasJFR
+ * @modules jdk.jfr/jdk.jfr.internal.test
+ * @run main/othervm -Xmx2g -XX:VMThreadStackSize=512 jdk.jfr.event.oldobject.TestDFSWithSmallStack dfsonly
+ */
+
+/**
+ * @test id=bfsdfs
+ * @summary Tests that DFS works with a small stack
+ * @library /test/lib /test/jdk
+ * @requires vm.hasJFR
+ * @modules jdk.jfr/jdk.jfr.internal.test
+ * @run main/othervm -Xmx2g -XX:VMThreadStackSize=512 jdk.jfr.event.oldobject.TestDFSWithSmallStack bfsdfs
+ */
+public class TestDFSWithSmallStack {
+
+    // Tests depth first search with a small stack.
+
+    // An non-zero exit code, together with a missing hs-err file or possibly a missing jfr file,
+    // indicates a native stack overflow happened and is a fail condition for this test.
+
+    // We build up an array of linked lists, each containing enough entries for DFS search to
+    // max out max_dfs_depth (but not greatly surpass it).
+
+    private static final int TOTAL_OBJECTS = 10_000_000;
+    private static final int OBJECTS_PER_LIST = 5_000;
+    public static LinkedList<Object>[] leak;
+
+    public static void main(String... args) throws Exception {
+
+        switch (args[0]) {
+            case "dfsonly" -> WhiteBox.setSkipBFS(true);
+            case "bfsdfs" -> {} /* ignored */
+            default -> throw new RuntimeException("Invalid argument");
+        }
+
+        WhiteBox.setWriteAllObjectSamples(true);
+        int count = 10;
+
+        while (count > 0) {
+            try (Recording r = new Recording()) {
+                r.enable(EventNames.OldObjectSample).with("cutoff", "infinity");
+                r.start();
+                leak = new LinkedList[TOTAL_OBJECTS / OBJECTS_PER_LIST];
+                for (int i = 0; i < leak.length; i++) {
+                    leak[i] = new LinkedList<Object>();
+                    for (int j = 0; j < OBJECTS_PER_LIST; j++) {
+                        leak[i].add(new Object());
+                    }
+                }
+                System.gc();
+                r.stop();
+                List<RecordedEvent> events = Events.fromRecording(r);
+                Events.hasEvents(events);
+                if (OldObjects.countChains(events) >= 30) {
+                    return;
+                }
+                System.out.println("Not enough chains found, retrying.");
+            }
+            count--;
+            leak = null;
+        }
+    }
+}