Fix NotifyPauseOnStart and NotifyPauseOnExit dead lock

runtime/vm/message_handler.cc

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/message_handler.h"
 
 #include "vm/dart.h"
 #include "vm/lockers.h"
 #include "vm/port.h"
 #include "vm/thread_interrupter.h"
@@ skipping 217 matching lines @@
 bool MessageHandler::HasOOBMessages() {
   MonitorLocker ml(&monitor_);
   return !oob_queue_->IsEmpty();
 }
 
 
 void MessageHandler::TaskCallback() {
   ASSERT(Isolate::Current() == NULL);
   bool ok = true;
   bool run_end_callback = false;
+  bool notify_paused_on_exit = false;
   {
     MonitorLocker ml(&monitor_);
     // Initialize the message handler by running its start function,
     // if we have one.  For an isolate, this will run the isolate's
     // main() function.
     if (pause_on_start()) {
       if (!paused_on_start_) {
+        // Temporarily drop the lock when calling out to NotifyPauseOnStart.
+        // This avoids a dead lock that can occur when this message handler
+        // tries to post a message while a message is being posted to it.
+        monitor_.Exit();
         NotifyPauseOnStart();
+        monitor_.Enter();
         paused_on_start_ = true;
       }
       HandleMessages(false, false);
       if (pause_on_start()) {
         // Still paused.
         task_ = NULL;  // No task in queue.
         return;
       } else {
         paused_on_start_ = false;
       }
@@ skipping 15 matching lines @@
     }
     task_ = NULL;  // No task in queue.
 
     if (!ok || !HasLivePorts()) {
       if (pause_on_exit()) {
         if (!paused_on_exit_) {
           if (FLAG_trace_service_pause_events) {
             OS::PrintErr("Isolate %s paused before exiting. "
                        "Use the Observatory to release it.\n", name());
           }
-          NotifyPauseOnExit();
+          notify_paused_on_exit = true;
           paused_on_exit_ = true;
         }
       } else {
         if (FLAG_trace_isolates) {
         OS::Print("[-] Stopping message handler (%s):\n"
                   "\thandler:    %s\n",
                   (ok ? "no live ports" : "error"),
                   name());
         }
         pool_ = NULL;
         run_end_callback = true;
         paused_on_exit_ = false;
       }
     }
   }
+  // At this point we no longer hold the message handler lock.
+  if (notify_paused_on_exit) {
+    NotifyPauseOnExit();
+  }
   if (run_end_callback && end_callback_ != NULL) {
     end_callback_(callback_data_);
     // The handler may have been deleted after this point.
   }
 }
 
 
 void MessageHandler::ClosePort(Dart_Port port) {
   MonitorLocker ml(&monitor_);
   if (FLAG_trace_isolates) {
@@ skipping 64 matching lines @@
 
 
 void MessageHandler::AcquireQueues(AcquiredQueues* acquired_queues) {
   ASSERT(acquired_queues != NULL);
   // No double dipping.
   ASSERT(acquired_queues->handler_ == NULL);
   acquired_queues->Reset(this);
 }
 
 }  // namespace dart