Added command line switches "crash-on-hang-threads" and "crash-on-hang-seconds"

chrome/browser/metrics/thread_watcher.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // This file defines a WatchDog thread that monitors the responsiveness of other
 // browser threads like UI, IO, DB, FILE and CACHED threads. It also defines
 // ThreadWatcher class which performs health check on threads that would like to
 // be watched. This file also defines ThreadWatcherList class that has list of
 // all active ThreadWatcher objects.
 //
 // ThreadWatcher class sends ping message to the watched thread and the watched
 // thread responds back with a pong message. It uploads response time
 // (difference between ping and pong times) as a histogram.
 //
 // TODO(raman): ThreadWatcher can detect hung threads. If a hung thread is
 // detected, we should probably just crash, and allow the crash system to gather
 // then stack trace.
 //
 // Example Usage:
 //
 //   The following is an example for watching responsiveness of IO thread.
 //   sleep_time specifies how often ping messages have to be sent to IO thread.
 //   unresponsive_time is the wait time after ping message is sent, to check if
 //   we have received pong message or not.
 //
 //   base::TimeDelta sleep_time = base::TimeDelta::FromSeconds(5);
 //   base::TimeDelta unresponsive_time = base::TimeDelta::FromSeconds(10);
+//   uint32 crash_on_unresponsive_count = ThreadWatcherList::kUnresponsiveCount;
+//   bool crash_on_hang = false;
 //   ThreadWatcher::StartWatching(BrowserThread::IO, "IO", sleep_time,
-//                                unresponsive_time);
+//                                unresponsive_time,
+//                                crash_on_unresponsive_count, crash_on_hang);
 
 #ifndef CHROME_BROWSER_METRICS_THREAD_WATCHER_H_
 #define CHROME_BROWSER_METRICS_THREAD_WATCHER_H_
 
 #include <map>
+#include <set>
 #include <string>
 #include <vector>
 
 #include "base/basictypes.h"
+#include "base/command_line.h"
 #include "base/gtest_prod_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/synchronization/lock.h"
 #include "base/task.h"
 #include "base/threading/thread.h"
 #include "base/time.h"
 #include "content/browser/browser_thread.h"
 #include "content/common/notification_observer.h"
 #include "content/common/notification_registrar.h"
 
 class CustomThreadWatcher;
 class ThreadWatcherList;
 
 // This class performs health check on threads that would like to be watched.
 class ThreadWatcher {
  public:
-  // This method starts performing health check on the given thread_id. It will
-  // create ThreadWatcher object for the given thread_id, thread_name,
-  // sleep_time and unresponsive_time. sleep_time_ is the wait time between ping
-  // messages. unresponsive_time_ is the wait time after ping message is sent,
-  // to check if we have received pong message or not. It will register that
-  // ThreadWatcher object and activate the thread watching of the given
-  // thread_id.
+  // This method starts performing health check on the given |thread_id|. It
+  // will create ThreadWatcher object for the given |thread_id|, |thread_name|.
+  // |sleep_time| is the wait time between ping messages. |unresponsive_time| is
+  // the wait time after ping message is sent, to check if we have received pong
+  // message or not. |crash_on_unresponsive_count| is used to determine if the
+  // thread is responsive or not. The watched thread is considered unresponsive
+  // if it hasn't responded with a pong message for
+  // |crash_on_unresponsive_count| number of ping messages. |crash_on_hang|
+  // specifies if browser should be crashed when the watched thread is
+  // unresponsive. It will register that ThreadWatcher object and activate the
+  // thread watching of the given thread_id.
   static void StartWatching(const BrowserThread::ID& thread_id,
                             const std::string& thread_name,
                             const base::TimeDelta& sleep_time,
-                            const base::TimeDelta& unresponsive_time);
+                            const base::TimeDelta& unresponsive_time,
+                            uint32 crash_on_unresponsive_count,
+                            bool crash_on_hang);
 
-  // Return the thread_id of the thread being watched.
+  // Return the |thread_id_| of the thread being watched.
   BrowserThread::ID thread_id() const { return thread_id_; }
 
   // Return the name of the thread being watched.
   std::string thread_name() const { return thread_name_; }
 
   // Return the sleep time between ping messages to be sent to the thread.
   base::TimeDelta sleep_time() const { return sleep_time_; }
 
   // Return the the wait time to check the responsiveness of the thread.
   base::TimeDelta unresponsive_time() const { return unresponsive_time_; }
 
   // Returns true if we are montioring the thread.
   bool active() const { return active_; }
 
-  // Returns ping_time_ (used by unit tests).
+  // Returns |ping_time_| (used by unit tests).
   base::TimeTicks ping_time() const { return ping_time_; }
 
-  // Returns ping_sequence_number_ (used by unit tests).
+  // Returns |ping_sequence_number_| (used by unit tests).
   uint64 ping_sequence_number() const { return ping_sequence_number_; }
 
  protected:
-  // Construct a ThreadWatcher for the given thread_id. sleep_time_ is the
-  // wait time between ping messages. unresponsive_time_ is the wait time after
+  // Construct a ThreadWatcher for the given |thread_id|. |sleep_time| is the
+  // wait time between ping messages. |unresponsive_time| is the wait time after
   // ping message is sent, to check if we have received pong message or not.
+  // |crash_on_unresponsive_count| is used to determine if the thread is
+  // responsive or not. The watched thread is considered unresponsive if it
+  // hasn't responded with a pong message for |crash_on_unresponsive_count|
+  // number of ping messages. |crash_on_hang| specifies if browser should be
+  // crashed when the watched thread is unresponsive.
   ThreadWatcher(const BrowserThread::ID& thread_id,
                 const std::string& thread_name,
                 const base::TimeDelta& sleep_time,
-                const base::TimeDelta& unresponsive_time);
+                const base::TimeDelta& unresponsive_time,
+                uint32 crash_on_unresponsive_count,
+                bool crash_on_hang);
   virtual ~ThreadWatcher();
 
   // This method activates the thread watching which starts ping/pong messaging.
   virtual void ActivateThreadWatching();
 
   // This method de-activates the thread watching and revokes all tasks.
   virtual void DeActivateThreadWatching();
 
   // This will ensure that the watching is actively taking place, and awaken
-  // (i.e., post a PostPingMessage) if the watcher has stopped pinging due to
-  // lack of user activity. It will also reset ping_count_ to kPingCount.
+  // (i.e., post a PostPingMessage()) if the watcher has stopped pinging due to
+  // lack of user activity. It will also reset |ping_count_| to |kPingCount|.
   virtual void WakeUp();
 
   // This method records when ping message was sent and it will Post a task
-  // (OnPingMessage) to the watched thread that does nothing but respond with
-  // OnPongMessage. It also posts a task (OnCheckResponsiveness) to check
+  // (OnPingMessage()) to the watched thread that does nothing but respond with
+  // OnPongMessage(). It also posts a task (OnCheckResponsiveness()) to check
   // responsiveness of monitored thread that would be called after waiting
-  // unresponsive_time_.
+  // |unresponsive_time_|.
   // This method is accessible on WatchDogThread.
   virtual void PostPingMessage();
 
   // This method handles a Pong Message from watched thread. It will track the
   // response time (pong time minus ping time) via histograms. It posts a
-  // PostPingMessage task that would be called after waiting sleep_time_.  It
-  // increments ping_sequence_number_ by 1.
+  // PostPingMessage() task that would be called after waiting |sleep_time_|. It
+  // increments |ping_sequence_number_| by 1.
   // This method is accessible on WatchDogThread.
   virtual void OnPongMessage(uint64 ping_sequence_number);
 
   // This method will determine if the watched thread is responsive or not. If
-  // the latest ping_sequence_number_ is not same as the ping_sequence_number
-  // that is passed in, then we can assume that watched thread has responded
-  // with a pong message.
+  // the latest |ping_sequence_number_| is not same as the
+  // |ping_sequence_number| that is passed in, then we can assume that watched
+  // thread has responded with a pong message.
   // This method is accessible on WatchDogThread.
   virtual bool OnCheckResponsiveness(uint64 ping_sequence_number);
 
  private:
   friend class ThreadWatcherList;
   friend class CustomThreadWatcher;
 
   // Allow tests to access our innards for testing purposes.
   FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, Registration);
   FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, ThreadResponding);
   FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, ThreadNotResponding);
   FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, MultipleThreadsResponding);
   FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, MultipleThreadsNotResponding);
 
   // Post constructor initialization.
   void Initialize();
 
   // Watched thread does nothing except post callback_task to the WATCHDOG
   // Thread. This method is called on watched thread.
   static void OnPingMessage(const BrowserThread::ID& thread_id,
                             Task* callback_task);
 
-  // This method resets unresponsive_count_ to zero because watched thread is
+  // This method resets |unresponsive_count_| to zero because watched thread is
   // responding to the ping message with a pong message.
   void ResetHangCounters();
 
   // This method records watched thread is not responding to the ping message.
-  // It increments unresponsive_count_ by 1.
+  // It increments |unresponsive_count_| by 1.
   void GotNoResponse();
 
+  // This method returns true if the watched thread has not responded with a
+  // pong message for |crash_on_unresponsive_count_| number of ping messages.
+  bool CrashOnUnresponsiveness();
+
   // This is the number of ping messages to be sent when the user is idle.
   // ping_count_ will be initialized to kPingCount whenever user becomes active.
   static const int kPingCount;
 
-  // This value is used to determine if the watched thread is responsive or not.
-  // If unresponsive_count_ is less than kUnresponsiveCount then watched thread
-  // is considered as responsive (in responsive_count_histogram_) otherwise it
-  // is considered as unresponsive (in unresponsive_count_histogram_).
-  static const int kUnresponsiveCount;
-
-  // The thread_id of the thread being watched. Only one instance can exist for
-  // the given thread_id of the thread being watched.
+  // The |thread_id_| of the thread being watched. Only one instance can exist
+  // for the given |thread_id_| of the thread being watched.
   const BrowserThread::ID thread_id_;
 
   // The name of the thread being watched.
   const std::string thread_name_;
 
   // It is the sleep time between between the receipt of a pong message back,
   // and the sending of another ping message.
   const base::TimeDelta sleep_time_;
 
   // It is the duration from sending a ping message, until we check status to be
@@ skipping 17 matching lines @@
   // The counter tracks least number of ping messages that will be sent to
   // watched thread before the ping-pong mechanism will go into an extended
   // sleep. If this value is zero, then the mechanism is in an extended sleep,
   // and awaiting some observed user action before continuing.
   int ping_count_;
 
   // Histogram that keeps track of response times for the watched thread.
   base::Histogram* response_time_histogram_;
 
   // Histogram that keeps track of unresponsive time since the last pong message
-  // when we got no response (GotNoResponse) from the watched thread.
+  // when we got no response (GotNoResponse()) from the watched thread.
   base::Histogram* unresponsive_time_histogram_;
 
   // Histogram that keeps track of how many threads are responding when we got
-  // no response (GotNoResponse) from the watched thread.
+  // no response (GotNoResponse()) from the watched thread.
   base::Histogram* responsive_count_histogram_;
 
   // Histogram that keeps track of how many threads are not responding when we
-  // got no response (GotNoResponse) from the watched thread. Count includes the
-  // thread that got no response.
+  // got no response (GotNoResponse()) from the watched thread. Count includes
+  // the thread that got no response.
   base::Histogram* unresponsive_count_histogram_;
 
   // This counter tracks the unresponsiveness of watched thread. If this value
   // is zero then watched thread has responded with a pong message. This is
-  // incremented by 1 when we got no response (GotNoResponse) from the watched
+  // incremented by 1 when we got no response (GotNoResponse()) from the watched
   // thread.
-  int unresponsive_count_;
+  uint32 unresponsive_count_;
 
   // This is set to true when we would have crashed the browser because the
   // watched thread hasn't responded atleast 6 times. It is reset to false when
   // watched thread responds with a pong message.
   bool hung_processing_complete_;
 
+  // This is used to determine if the watched thread is responsive or not. If
+  // watched thread's |unresponsive_count_| is greater than or equal to
+  // |crash_on_unresponsive_count_| then we would consider it as unresponsive.
+  uint32 crash_on_unresponsive_count_;

[jar (doing other things), 2011/06/14 00:56:27]

nit: Consider changing the name.  The count is about a threshold, but it may or
may not involve crashing.  How about:

unresponsive_threshold_

[ramant (doing other things), 2011/06/16 22:26:45]

Done.

+
+  // This is set to true if we want to the crash the browser when watched
+  // thread has not responded with a pong message for
+  // |crash_on_unresponsive_count_| number of ping messages.

[jar (doing other things), 2011/06/14 00:56:27]

Some of the comment are repetetive.  It is nicer when possible to refer to other
items for details.  For instance, this could be:

This is set to true if we want to crash the browser when the watch thread has
become sufficiently unresponsive, while other threads are sufficiently
responsive.


The advantage of this is that IF we change something, we don't need to change a
ton of comments.  It best, these little micro-comments could duplicate a header
comment, which describes the whole picture.  You shouldn't have to keep
mentioning the count in each of these comments though.  Also note that it is
more than count, as we want a couple of things to to have happened.

[ramant (doing other things), 2011/06/16 22:26:45]

Done.

+  bool crash_on_hang_;
+
   // We use this factory to create callback tasks for ThreadWatcher object. We
   // use this during ping-pong messaging between WatchDog thread and watched
   // thread.
   ScopedRunnableMethodFactory<ThreadWatcher> method_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(ThreadWatcher);
 };
 
 // Class with a list of all active thread watchers.  A thread watcher is active
 // if it has been registered, which includes determing the histogram name. This
 // class provides utility functions to start and stop watching all browser
 // threads. Only one instance of this class exists.
 class ThreadWatcherList : public NotificationObserver {
  public:
   // A map from BrowserThread to the actual instances.
   typedef std::map<BrowserThread::ID, ThreadWatcher*> RegistrationList;
 
   // This singleton holds the global list of registered ThreadWatchers.
-  ThreadWatcherList();
+  explicit ThreadWatcherList(const CommandLine& command_line);
   // Destructor deletes all registered ThreadWatcher instances.
   virtual ~ThreadWatcherList();
 
   // Register() stores a pointer to the given ThreadWatcher in a global map.
   static void Register(ThreadWatcher* watcher);
 
   // This method returns true if the ThreadWatcher object is registerd.
   static bool IsRegistered(const BrowserThread::ID thread_id);
 
   // This method posts a task on WatchDogThread to start watching all browser
-  // threads.
+  // threads. It calls ThreadWatcherList::StartWatching() on each browser thread
+  // that is watched.
   // This method is accessible on UI thread.
   static void StartWatchingAll();
 
   // This method posts a task on WatchDogThread to RevokeAll tasks and to
   // deactive thread watching of other threads and tell NotificationService to
   // stop calling Observe.
   // This method is accessible on UI thread.
   static void StopWatchingAll();
 
   // RemoveAll NotificationTypes that are being observed.
   // This method is accessible on UI thread.
   static void RemoveNotifications();
 
-  // This method returns number of watched threads that have responded and
-  // threads that have not responded with a pong message.
+  // This method returns number of responsive and unresponsive watched threads.
   static void GetStatusOfThreads(int* no_of_responding_threads,

[jar (doing other things), 2011/06/14 00:56:27]

nit: abreviations should not be used in names in this case "no" means "number." 
Suggest:
responding_thread_count
unresponding_thread_count

[ramant (doing other things), 2011/06/16 22:26:45]

Done.

                                  int* no_of_unresponding_threads);
 
  private:
   // Allow tests to access our innards for testing purposes.
+  friend class CustomThreadWatcher;
+  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, CommandLineArgs);
   FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, Registration);
 
+  // This method calls ThreadWatcher::StartWatching() to perform health check on
+  // the given |thread_id|, |thread_name|. |sleep_time| is the wait time between
+  // ping messages. |unresponsive_time| is the wait time after ping message is
+  // sent, to check if we have received pong message or not.
+  static void StartWatching(const BrowserThread::ID& thread_id,
+                            const std::string& thread_name,
+                            const base::TimeDelta& sleep_time,
+                            const base::TimeDelta& unresponsive_time);
+
+  // Helper methods to access crash_on_unresponsive_count_ (for unit tests).
+  uint32 CrashOnUnresponsiveCount();
+  uint32 PreLockedCrashOnUnresponsiveCount() const {
+      return crash_on_unresponsive_count_;
+  }
+
+  // Returns true if the |thread_name| is in the |crash_on_hang_thread_names_|.
+  bool CrashOnHang(const std::string& thread_name);
+  bool PreLockedCrashOnHang(const std::string& thread_name) const;
+
   // Delete all thread watcher objects and remove them from global map.
   // This method is accessible on WatchDogThread.
   void DeleteAll();
 
   // This will ensure that the watching is actively taking place. It will wakeup
   // all thread watchers every 2 seconds. This is the implementation of
   // NotificationObserver. When a matching notification is posted to the
   // notification service, this method is called.
   // This method is accessible on UI thread.
   virtual void Observe(NotificationType type,
@@ skipping 14 matching lines @@
 
   static ThreadWatcherList* global_;  // The singleton of this class.
 
   // This is the wait time between ping messages.
   static const int kSleepSeconds;
 
   // This is the wait time after ping message is sent, to check if we have
   // received pong message or not.
   static const int kUnresponsiveSeconds;
 
+  // Default values for |crash_on_unresponsive_count_|.
+  static const int kUnresponsiveCount;
+
   // Lock for access to registered_.
   base::Lock lock_;
 
   // Map of all registered watched threads, from thread_id to ThreadWatcher.
   RegistrationList registered_;
 
   // The registrar that holds NotificationTypes to be observed.
   NotificationRegistrar registrar_;
 
   // This is the last time when woke all thread watchers up.
   base::TimeTicks last_wakeup_time_;
 
+  // This is used to determine if the watched thread is responsive or not. If
+  // watched thread's |unresponsive_count_| is greater than or equal to
+  // |crash_on_unresponsive_count_| then we could crash the browser if the
+  // watched thread is in |crash_on_hang_thread_names_|.
+  // |crash_on_unresponsive_count_| is initialized with |kUnresponsiveCount|,
+  // but can be overwritten by the command line switch
+  // "--crash-on-hang-seconds".
+  uint32 crash_on_unresponsive_count_;
+
+  // This is the set of watched thread's names that are to be crashed if they
+  // have not responded with a pong message for |crash_on_unresponsive_count_|
+  // number of ping messages.
+  std::set<std::string> crash_on_hang_thread_names_;
+
   DISALLOW_COPY_AND_ASSIGN(ThreadWatcherList);
 };
 
 // Class for WatchDogThread and in its Init method, we start watching UI, IO,
 // DB, FILE, CACHED threads.
 class WatchDogThread : public base::Thread {
  public:
   // Constructor.
   WatchDogThread();
 
@@ skipping 32 matching lines @@
 
   DISALLOW_COPY_AND_ASSIGN(WatchDogThread);
 };
 
 // DISABLE_RUNNABLE_METHOD_REFCOUNT is a convenience macro for disabling
 // refcounting of ThreadWatcher and ThreadWatcherList classes.
 DISABLE_RUNNABLE_METHOD_REFCOUNT(ThreadWatcher);
 DISABLE_RUNNABLE_METHOD_REFCOUNT(ThreadWatcherList);
 
 #endif  // CHROME_BROWSER_METRICS_THREAD_WATCHER_H_