[heap, debugger] Introduce out-of-memory listener for debugger.

test/cctest/test-debug.cc

Index: test/cctest/test-debug.cc
diff --git a/test/cctest/test-debug.cc b/test/cctest/test-debug.cc
index 5af65cb1082e52a1e21c630d39d764084d2b22eb..33b528ef1f0162ed402d6e44d36ba554ede60814 100644
--- a/test/cctest/test-debug.cc
+++ b/test/cctest/test-debug.cc
@@ -6665,3 +6665,33 @@ TEST(DebugStepOverFunctionWithCaughtException) {
   v8::Debug::SetDebugEventListener(env->GetIsolate(), nullptr);
   CHECK_EQ(break_point_hit_count, 4);
 }
+
+int out_of_memory_callback_data = 10;
+bool out_of_memory_callback_called = false;
+void OutOfMemoryCallback(v8::Isolate* isolate, void* data) {
+  CHECK_EQ(data, &out_of_memory_callback_data);
+  out_of_memory_callback_called = true;
+  isolate->IncreaseHeapLimitForDebugging();
+}
+
+UNINITIALIZED_TEST(DebugSetOutOfMemoryListener) {
+  v8::Isolate::CreateParams create_params;
+  create_params.array_buffer_allocator = CcTest::array_buffer_allocator();
+  create_params.constraints.set_max_old_space_size(10);
+  v8::Isolate* isolate = v8::Isolate::New(create_params);
+  i::Isolate* i_isolate = reinterpret_cast<i::Isolate*>(isolate);
+  {
+    v8::Isolate::Scope i_scope(isolate);
+    v8::HandleScope scope(isolate);
+    LocalContext context(isolate);
+    v8::debug::SetOutOfMemoryCallback(isolate, OutOfMemoryCallback,
+                                      &out_of_memory_callback_data);
+    CHECK(!out_of_memory_callback_called);
+    // The following allocation fails unless the out-of-memory callback
+    // increases the heap limit.
+    int length = 10 * i::MB / i::kPointerSize;
+    i_isolate->factory()->NewFixedArray(length, i::TENURED);
+    CHECK(out_of_memory_callback_called);
+  }
+  isolate->Dispose();
+}