Provide a mechanism for dumping of heap profiles on predefined events.

runtime/vm/isolate.cc

Index: runtime/vm/isolate.cc
diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc
index 6885ac03b96595b3776c4abcce391f29e94bcd7b..3e86dc1a271c9a669e6f8b2989f313cb89881cbf 100644
--- a/runtime/vm/isolate.cc
+++ b/runtime/vm/isolate.cc
@@ -30,6 +30,7 @@ DEFINE_FLAG(bool, report_usage_count, false,
             "Track function usage and report.");
 DEFINE_FLAG(bool, trace_isolates, false,
             "Trace isolate creation and shut down.");
+DECLARE_FLAG(bool, heap_profile_out_of_memory);
 DECLARE_FLAG(bool, generate_gdb_symbols);
 
 
@@ -114,6 +115,18 @@ bool IsolateMessageHandler::HandleMessage(Message* message) {
       if (result.IsError()) {
         isolate_->object_store()->set_sticky_error(Error::Cast(result));
       }
+      if (result.IsInstance() && FLAG_heap_profile_out_of_memory) {

[siva, 2012/12/07 00:33:02]

This check seems incorrect, at this point result will
be an UnhandledException if an OOM was encountered while handling the message.

You can then do something like:
const Instance& exception = Instance::Handle(result.exception());
if (exception.raw() == isolate->object_store()->out_of_memory()) {
   ....
}

[cshapiro, 2012/12/08 07:29:46]

I am really confused by this.  At this point result can be an Error or
UnhandledException but the handle type is Object.

I think what you are suggesting is that if an OOM is thrown, it will be wrapped
in an UnhandledException.  However, since any Error is admitted, it seems
possible for an OOM to be passed as an argument as well.

What is doubly confusing is that UnhandledException is a subclass of Error so
the result.IsUnhandledException() test seems pointless.

Anyway, I have handled both cases explicitly.  If I have got it wrong, I can
easily prune either case.

+        Instance& out_of_memory =
+            Instance::Handle(isolate_->object_store()->out_of_memory());
+        if (Instance::Cast(result).Equals(out_of_memory)) {
+          const char* format = "%s-out-of-memory.hprof";
+          intptr_t len = OS::SNPrint(NULL, 0, format, isolate_->name());
+          char* filename =
+              isolate_->current_zone()->Alloc<char>(len + 1);
+          OS::SNPrint(filename, len + 1, format, isolate_->name());
+          isolate_->heap()->ProfileToFile(filename);
+        }
+      }
       if (Isolate::UnhandledExceptionCallback() != NULL) {
         Dart_EnterScope();
         Dart_Handle error = Api::NewHandle(isolate_, result.raw());