Added command line switches "crash-on-hang-threads" and "crash-on-hang-seconds"

chrome/browser/metrics/thread_watcher.cc

Index: chrome/browser/metrics/thread_watcher.cc
===================================================================
--- chrome/browser/metrics/thread_watcher.cc	(revision 89361)
+++ chrome/browser/metrics/thread_watcher.cc	(working copy)
@@ -2,10 +2,14 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include <math.h>  // ceil
+
+#include "base/string_tokenizer.h"
 #include "base/threading/thread_restrictions.h"
 #include "build/build_config.h"
 #include "chrome/browser/metrics/metrics_service.h"
 #include "chrome/browser/metrics/thread_watcher.h"
+#include "chrome/common/chrome_switches.h"
 #include "content/common/notification_service.h"
 
 #if defined(OS_WIN)
@@ -15,14 +19,14 @@
 // static
 const int ThreadWatcher::kPingCount = 6;
 
-// static
-const int ThreadWatcher::kUnresponsiveCount = 6;
-
 // ThreadWatcher methods and members.
 ThreadWatcher::ThreadWatcher(const BrowserThread::ID& thread_id,
                              const std::string& thread_name,
                              const base::TimeDelta& sleep_time,
-                             const base::TimeDelta& unresponsive_time)
+                             const base::TimeDelta& unresponsive_time,
+                             uint32 unresponsive_threshold,
+                             bool crash_on_hang,
+                             uint32 live_threads_threshold)
     : thread_id_(thread_id),
       thread_name_(thread_name),
       sleep_time_(sleep_time),
@@ -36,7 +40,11 @@
       unresponsive_time_histogram_(NULL),
       unresponsive_count_(0),
       hung_processing_complete_(false),
+      unresponsive_threshold_(unresponsive_threshold),
+      crash_on_hang_(crash_on_hang),
+      live_threads_threshold_(live_threads_threshold),
       ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)) {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
   Initialize();
 }
 
@@ -46,7 +54,10 @@
 void ThreadWatcher::StartWatching(const BrowserThread::ID& thread_id,
                                   const std::string& thread_name,
                                   const base::TimeDelta& sleep_time,
-                                  const base::TimeDelta& unresponsive_time) {
+                                  const base::TimeDelta& unresponsive_time,
+                                  uint32 unresponsive_threshold,
+                                  bool crash_on_hang,
+                                  uint32 live_threads_threshold) {
   DCHECK_GE(sleep_time.InMilliseconds(), 0);
   DCHECK_GE(unresponsive_time.InMilliseconds(), sleep_time.InMilliseconds());
 
@@ -55,17 +66,27 @@
   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
     WatchDogThread::PostTask(
         FROM_HERE,
-        NewRunnableFunction(
-            &ThreadWatcher::StartWatching,
-            thread_id, thread_name, sleep_time, unresponsive_time));
+        NewRunnableFunction(&ThreadWatcher::StartWatching,
+                            thread_id,
+                            thread_name,
+                            sleep_time,
+                            unresponsive_time,
+                            unresponsive_threshold,
+                            crash_on_hang,
+                            live_threads_threshold));
     return;
   }
 
   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 
   // Create a new thread watcher object for the given thread and activate it.
-  ThreadWatcher* watcher =
-      new ThreadWatcher(thread_id, thread_name, sleep_time, unresponsive_time);
+  ThreadWatcher* watcher = new ThreadWatcher(thread_id,
+                                             thread_name,
+                                             sleep_time,
+                                             unresponsive_time,
+                                             unresponsive_threshold,
+                                             crash_on_hang,
+                                             live_threads_threshold);
   DCHECK(watcher);
   // If we couldn't register the thread watcher object, we are shutting down,
   // then don't activate thread watching.
@@ -196,6 +217,7 @@
 }
 
 void ThreadWatcher::Initialize() {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
   ThreadWatcherList::Register(this);
 
   const std::string response_time_histogram_name =
@@ -244,9 +266,8 @@
 void ThreadWatcher::GotNoResponse() {
   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 
-  // Record how other threads are responding when we don't get a response for
-  // ping message atleast kUnresponsiveCount times.
-  if (++unresponsive_count_ < kUnresponsiveCount)
+  ++unresponsive_count_;
+  if (!CrashOnUnresponsiveness())
     return;
 
   // Record total unresponsive_time since last pong message.
@@ -257,210 +278,317 @@
   if (hung_processing_complete_)
     return;
 
-  int no_of_responding_threads = 0;
-  int no_of_unresponding_threads = 0;
-  ThreadWatcherList::GetStatusOfThreads(&no_of_responding_threads,
-                                        &no_of_unresponding_threads);
+  // Record how other threads are responding.
+  uint32 responding_thread_count = 0;
+  uint32 unresponding_thread_count = 0;
+  ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
+                                        &unresponding_thread_count);
 
   // Record how many watched threads are responding.
-  responsive_count_histogram_->Add(no_of_responding_threads);
+  responsive_count_histogram_->Add(responding_thread_count);
 
   // Record how many watched threads are not responding.
-  unresponsive_count_histogram_->Add(no_of_unresponding_threads);
+  unresponsive_count_histogram_->Add(unresponding_thread_count);
 
-  // Crash the browser if IO thread hasn't responded atleast kUnresponsiveCount
-  // times and if the number of other threads is equal to 1. We picked 1 to
-  // reduce the number of crashes and to get some sample data.
-  if (thread_id_ == BrowserThread::IO && no_of_responding_threads == 1) {
+  // Crash the browser if the watched thread is to be crashed on hang and if the
+  // number of other threads responding is equal to live_threads_threshold_.
+  if (crash_on_hang_ && responding_thread_count == live_threads_threshold_) {
     int* crash = NULL;
-    CHECK(crash++);
+    CHECK(crash+thread_id_);

[jar (doing other things), 2011/06/18 15:04:16]

style nit: spaces around binary operator "+"

[ramant (doing other things), 2011/06/19 21:18:16]

Done.

   }
 
   hung_processing_complete_ = true;
 }
 
+bool ThreadWatcher::CrashOnUnresponsiveness() {

[jar (doing other things), 2011/06/18 15:04:16]

nit: The function name almost suggests this function does the crashing.  Better
may be:

IsVeryUnresponsive()

The "Very" will hint at "beyond a threshold"

[ramant (doing other things), 2011/06/19 21:18:16]

Done.

+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+  return unresponsive_count_ >= unresponsive_threshold_;
+}
+
 // ThreadWatcherList methods and members.
 //
 // static
-ThreadWatcherList* ThreadWatcherList::global_ = NULL;
+ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
 // static
 const int ThreadWatcherList::kSleepSeconds = 1;
 // static
 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
+// static
+const int ThreadWatcherList::kUnresponsiveCount = 6;
+// static
+const int ThreadWatcherList::kLiveThreadsThreshold = 1;
 
-ThreadWatcherList::ThreadWatcherList()
-    : last_wakeup_time_(base::TimeTicks::Now()) {
-  // Assert we are not running on WATCHDOG thread. Would be ideal to assert we
-  // are on UI thread, but Unit tests are not running on UI thread.
-  DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
-  CHECK(!global_);
-  global_ = this;
-  // Register Notifications observer.
-  MetricsService::SetUpNotifications(&registrar_, this);
+// static
+void ThreadWatcherList::StartWatchingAll(const CommandLine& command_line) {
+  ThreadWatcherObserver::SetupNotifications(
+      base::TimeDelta::FromSeconds(kSleepSeconds));
+
+  uint32 unresponsive_threshold;
+  std::set<std::string> crash_on_hang_thread_names;
+  uint32 live_threads_threshold;
+  ParseCommandLine(command_line,
+                   &unresponsive_threshold,
+                   &crash_on_hang_thread_names,
+                   &live_threads_threshold);
+
+  WatchDogThread::PostDelayedTask(
+      FROM_HERE,
+      NewRunnableFunction(&ThreadWatcherList::InitializeAndStartWatching,
+                          unresponsive_threshold,
+                          crash_on_hang_thread_names,
+                          live_threads_threshold),
+      base::TimeDelta::FromSeconds(120).InMilliseconds());
 }
 
-ThreadWatcherList::~ThreadWatcherList() {
-  base::AutoLock auto_lock(lock_);
-  DCHECK(this == global_);
-  global_ = NULL;
+// static
+void ThreadWatcherList::StopWatchingAll() {
+  ThreadWatcherObserver::RemoveNotifications();
+  DeleteAll();
 }
 
 // static
 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
-  if (!global_)
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+  if (!g_thread_watcher_list_)
     return;
-  base::AutoLock auto_lock(global_->lock_);
-  DCHECK(!global_->PreLockedFind(watcher->thread_id()));
-  global_->registered_[watcher->thread_id()] = watcher;
+  DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
+  g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 }
 
 // static
 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
   return NULL != ThreadWatcherList::Find(thread_id);
 }
 
 // static
-void ThreadWatcherList::StartWatchingAll() {
-  if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
-    WatchDogThread::PostDelayedTask(
-        FROM_HERE,
-        NewRunnableFunction(&ThreadWatcherList::StartWatchingAll),
-        base::TimeDelta::FromSeconds(120).InMilliseconds());
+void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
+                                           uint32* unresponding_thread_count) {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+  *responding_thread_count = 0;
+  *unresponding_thread_count = 0;
+  if (!g_thread_watcher_list_)
     return;
+
+  for (RegistrationList::iterator it =
+           g_thread_watcher_list_->registered_.begin();
+       g_thread_watcher_list_->registered_.end() != it;
+       ++it) {
+    if (it->second->CrashOnUnresponsiveness())
+      ++(*unresponding_thread_count);
+    else
+      ++(*responding_thread_count);
   }
+}
+
+// static
+void ThreadWatcherList::WakeUpAll() {
   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+  if (!g_thread_watcher_list_)
+    return;
+
+  for (RegistrationList::iterator it =
+           g_thread_watcher_list_->registered_.begin();
+       g_thread_watcher_list_->registered_.end() != it;
+       ++it)
+    it->second->WakeUp();
+}
+
+ThreadWatcherList::ThreadWatcherList() {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+  CHECK(!g_thread_watcher_list_);
+  g_thread_watcher_list_ = this;
+}
+
+ThreadWatcherList::~ThreadWatcherList() {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+  DCHECK(this == g_thread_watcher_list_);
+  g_thread_watcher_list_ = NULL;
+}
+
+// static
+void ThreadWatcherList::ParseCommandLine(
+    const CommandLine& command_line,
+    uint32* unresponsive_threshold,
+    std::set<std::string>* crash_on_hang_thread_names,
+    uint32* live_threads_threshold) {
+  // Determine |unresponsive_threshold| based on switches::kCrashOnHangSeconds.
+  *unresponsive_threshold = kUnresponsiveCount;
+  std::string crash_on_hang_seconds =
+      command_line.GetSwitchValueASCII(switches::kCrashOnHangSeconds);
+  if (!crash_on_hang_seconds.empty()) {
+    int crash_seconds = atoi(crash_on_hang_seconds.c_str());
+    if (crash_seconds > 0) {
+      *unresponsive_threshold = static_cast<uint32>(
+          ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
+    }
+  }
+
+  // Default to crashing the browser if UI or IO threads are not responsive.
+  std::string crash_on_hang_threads =  "UI,IO";
+  if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
+    crash_on_hang_threads =
+        command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
+  }
+  StringTokenizer tokens(crash_on_hang_threads, ",");
+  while (tokens.GetNext())
+    crash_on_hang_thread_names->insert(tokens.token());
+
+  // Determine |live_threads_threshold| based on switches::kCrashOnLive.
+  *live_threads_threshold = kLiveThreadsThreshold;
+  if (command_line.HasSwitch(switches::kCrashOnLive)) {
+    std::string live_threads =
+        command_line.GetSwitchValueASCII(switches::kCrashOnLive);
+    *live_threads_threshold = static_cast<uint32>(atoi(live_threads.c_str()));
+  }
+}
+
+// static
+void ThreadWatcherList::InitializeAndStartWatching(
+    uint32 unresponsive_threshold,
+    const std::set<std::string>& crash_on_hang_thread_names,
+    uint32 live_threads_threshold) {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+
+  ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
+  CHECK(thread_watcher_list);
+
   const base::TimeDelta kSleepTime =
       base::TimeDelta::FromSeconds(kSleepSeconds);
   const base::TimeDelta kUnresponsiveTime =
       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
-  if (BrowserThread::IsMessageLoopValid(BrowserThread::UI)) {
-    ThreadWatcher::StartWatching(BrowserThread::UI, "UI", kSleepTime,
-                                 kUnresponsiveTime);
-  }
-  if (BrowserThread::IsMessageLoopValid(BrowserThread::IO)) {
-    ThreadWatcher::StartWatching(BrowserThread::IO, "IO", kSleepTime,
-                                 kUnresponsiveTime);
-  }
-  if (BrowserThread::IsMessageLoopValid(BrowserThread::DB)) {
-    ThreadWatcher::StartWatching(BrowserThread::DB, "DB", kSleepTime,
-                                 kUnresponsiveTime);
-  }
-  if (BrowserThread::IsMessageLoopValid(BrowserThread::FILE)) {
-    ThreadWatcher::StartWatching(BrowserThread::FILE, "FILE", kSleepTime,
-                                 kUnresponsiveTime);
-  }
-  if (BrowserThread::IsMessageLoopValid(BrowserThread::CACHE)) {
-    ThreadWatcher::StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime,
-                                 kUnresponsiveTime);
-  }
+
+  StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
+                unresponsive_threshold, crash_on_hang_thread_names,
+                live_threads_threshold);
+  StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
+                unresponsive_threshold, crash_on_hang_thread_names,
+                live_threads_threshold);
+  StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
+                unresponsive_threshold, crash_on_hang_thread_names,
+                live_threads_threshold);
+  StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
+                unresponsive_threshold, crash_on_hang_thread_names,
+                live_threads_threshold);
+  StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
+                unresponsive_threshold, crash_on_hang_thread_names,
+                live_threads_threshold);
 }
 
 // static
-void ThreadWatcherList::StopWatchingAll() {
-  // Assert we are not running on WATCHDOG thread. Would be ideal to assert we
-  // are on UI thread, but Unit tests are not running on UI thread.
-  DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
-  if (!global_)
+void ThreadWatcherList::StartWatching(
+    const BrowserThread::ID& thread_id,
+    const std::string& thread_name,
+    const base::TimeDelta& sleep_time,
+    const base::TimeDelta& unresponsive_time,
+    uint32 unresponsive_threshold,
+    const std::set<std::string>& crash_on_hang_thread_names,
+    uint32 live_threads_threshold) {
+  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+
+  if (!BrowserThread::IsMessageLoopValid(thread_id))
     return;
 
-  // Remove all notifications for all watched threads.
-  RemoveNotifications();
+  std::set<std::string>::const_iterator it =
+      crash_on_hang_thread_names.find(thread_name);
+  bool crash_on_hang = (it != crash_on_hang_thread_names.end());
 
-  // Delete all thread watcher objects on WatchDogThread.
-  WatchDogThread::PostTask(
-      FROM_HERE,
-      NewRunnableMethod(global_, &ThreadWatcherList::DeleteAll));
+  ThreadWatcher::StartWatching(thread_id,
+                               thread_name,
+                               sleep_time,
+                               unresponsive_time,
+                               unresponsive_threshold,
+                               crash_on_hang,
+                               live_threads_threshold);
 }
 
 // static
-void ThreadWatcherList::RemoveNotifications() {
-  // Assert we are not running on WATCHDOG thread. Would be ideal to assert we
-  // are on UI thread, but Unit tests are not running on UI thread.
-  DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
-  if (!global_)
+void ThreadWatcherList::DeleteAll() {
+  if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
+    WatchDogThread::PostTask(
+        FROM_HERE,
+        NewRunnableFunction(&ThreadWatcherList::DeleteAll));
     return;
-  base::AutoLock auto_lock(global_->lock_);
-  global_->registrar_.RemoveAll();
-}
+  }
 
-// static
-void ThreadWatcherList::GetStatusOfThreads(int* no_of_responding_threads,
-                                           int* no_of_unresponding_threads) {
   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
-  *no_of_responding_threads = 0;
-  *no_of_unresponding_threads = 0;
-  if (!global_)
+  if (!g_thread_watcher_list_)
     return;
 
-  base::AutoLock auto_lock(global_->lock_);
-  for (RegistrationList::iterator it = global_->registered_.begin();
-       global_->registered_.end() != it;
-       ++it) {
-    if (it->second->unresponsive_count_ < ThreadWatcher::kUnresponsiveCount)
-      ++(*no_of_responding_threads);
-    else
-      ++(*no_of_unresponding_threads);
+  // Delete all thread watcher objects.
+  while (!g_thread_watcher_list_->registered_.empty()) {
+    RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
+    delete it->second;
+    g_thread_watcher_list_->registered_.erase(it->first);

[jar (doing other things), 2011/06/18 15:04:16]

nit: Not a big deal for perf here.... but you might be able to just do:
erase(it)
rather than
erase(it->first)

[ramant (doing other things), 2011/06/19 21:18:16]

Done.

   }
+
+  delete g_thread_watcher_list_;
 }
 
-void ThreadWatcherList::DeleteAll() {
+// static
+ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
-  base::AutoLock auto_lock(lock_);
-  while (!registered_.empty()) {
-    RegistrationList::iterator it = registered_.begin();
-    delete it->second;
-    registered_.erase(it->first);
-  }
+  if (!g_thread_watcher_list_)
+    return NULL;
+  RegistrationList::iterator it =
+      g_thread_watcher_list_->registered_.find(thread_id);
+  if (g_thread_watcher_list_->registered_.end() == it)
+    return NULL;
+  return it->second;
 }
 
-void ThreadWatcherList::Observe(NotificationType type,
-                                const NotificationSource& source,
-                                const NotificationDetails& details) {
+// ThreadWatcherObserver methods and members.
+//
+// static
+ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
+
+ThreadWatcherObserver::ThreadWatcherObserver(
+    const base::TimeDelta& wakeup_interval)
+    : last_wakeup_time_(base::TimeTicks::Now()),
+      wakeup_interval_(wakeup_interval) {
+  CHECK(!g_thread_watcher_observer_);
+  g_thread_watcher_observer_ = this;
+}
+
+ThreadWatcherObserver::~ThreadWatcherObserver() {
+  DCHECK(this == g_thread_watcher_observer_);
+  g_thread_watcher_observer_ = NULL;
+}
+
+// static
+void ThreadWatcherObserver::SetupNotifications(
+    const base::TimeDelta& wakeup_interval) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
+  MetricsService::SetUpNotifications(&observer->registrar_, observer);
+}
+
+// static
+void ThreadWatcherObserver::RemoveNotifications() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  if (!g_thread_watcher_observer_)
+    return;
+  g_thread_watcher_observer_->registrar_.RemoveAll();
+  delete g_thread_watcher_observer_;
+}
+
+void ThreadWatcherObserver::Observe(NotificationType type,
+                                    const NotificationSource& source,
+                                    const NotificationDetails& details) {
   // There is some user activity, see if thread watchers are to be awakened.
   bool need_to_awaken = false;
   base::TimeTicks now = base::TimeTicks::Now();
-  {
-    base::AutoLock lock(lock_);
-    if (now - last_wakeup_time_ > base::TimeDelta::FromSeconds(kSleepSeconds)) {
-      need_to_awaken = true;
-      last_wakeup_time_ = now;
-    }
+  if ((now - last_wakeup_time_) > wakeup_interval_) {
+    need_to_awaken = true;

[jar (doing other things), 2011/06/18 15:04:16]

nit: Early return would make this cleaner.  You won't need the bool.
if (now - last_wakeup_time_ < wakeup_interval)
  return;

[ramant (doing other things), 2011/06/19 21:18:16]

Done.

+    last_wakeup_time_ = now;
   }
   if (need_to_awaken) {
     WatchDogThread::PostTask(
         FROM_HERE,
-        NewRunnableMethod(this, &ThreadWatcherList::WakeUpAll));
+        NewRunnableFunction(&ThreadWatcherList::WakeUpAll));
   }
 }
 
-void ThreadWatcherList::WakeUpAll() {
-  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
-  if (!global_)
-    return;
-  base::AutoLock auto_lock(lock_);
-  for (RegistrationList::iterator it = global_->registered_.begin();
-       global_->registered_.end() != it;
-       ++it)
-    it->second->WakeUp();
-}
-
-// static
-ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
-  if (!global_)
-    return NULL;
-  base::AutoLock auto_lock(global_->lock_);
-  return global_->PreLockedFind(thread_id);
-}
-
-ThreadWatcher* ThreadWatcherList::PreLockedFind(
-    const BrowserThread::ID& thread_id) {
-  RegistrationList::iterator it = registered_.find(thread_id);
-  if (registered_.end() == it)
-    return NULL;
-  return it->second;
-}
-
 // WatchDogThread methods and members.
 //
 // static