GPU Watchdog to check I/O before terminating GPU process

content/gpu/gpu_watchdog_thread.cc

Index: content/gpu/gpu_watchdog_thread.cc
diff --git a/content/gpu/gpu_watchdog_thread.cc b/content/gpu/gpu_watchdog_thread.cc
index dddb40945af57816ceb155bd932c7a3603cfe69d..e59deba3d3509af57793bef6d4553d54794e5aff 100644
--- a/content/gpu/gpu_watchdog_thread.cc
+++ b/content/gpu/gpu_watchdog_thread.cc
@@ -12,6 +12,7 @@
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/debug/alias.h"
+#include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/location.h"
 #include "base/macros.h"
@@ -19,6 +20,7 @@
 #include "base/process/process.h"
 #include "base/single_thread_task_runner.h"
 #include "base/threading/platform_thread.h"
+#include "base/timer/elapsed_timer.h"
 #include "build/build_config.h"
 #include "content/public/common/content_switches.h"
 #include "content/public/common/result_codes.h"
@@ -51,6 +53,7 @@ GpuWatchdogThread::GpuWatchdogThread(int timeout)
       arm_cpu_time_(),
 #endif
       suspended_(false),
+      use_temp_file_for_io_checking_(false),
 #if defined(USE_X11)
       display_(NULL),
       window_(0),
@@ -208,6 +211,8 @@ void GpuWatchdogThread::OnCheck(bool after_suspend) {
   // miss the false -> true transition.
   armed_ = true;
 
+  use_temp_file_for_io_checking_ = true;
+
 #if defined(OS_WIN)
   arm_cpu_time_ = GetWatchedThreadTime();
 
@@ -267,6 +272,42 @@ void GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang() {
     return;
   }
 
+  // If the watchdog woke up while there is a heavy I/O we want to give the
+  // machine a chance to clear the I/O queue before terminating the process.
+  if (use_temp_file_for_io_checking_) {

[jbauman, 2016/05/17 22:57:20]

Instead of using a flag here, we should make this into a separate method.
Everything that currently does a PostTask of
GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang would instead do a
posttask of the new method (GpuWatchdogThread::CheckForIo ?), and this new
method would be the only thing to PostTask
GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang.

[stanisc, 2016/05/18 00:01:30]

Good point! I didn't realize that adding sandbox rules works only on Windows. I
might have to go back to the first iteration where I created the temp file
before lowering the sandbox (see patch set 1). Another option is to limit this
to Windows only.

+    base::File temp_file;
+    base::FilePath temp_dir_path;
+    if (GetTempDir(&temp_dir_path)) {
+      base::FilePath temp_file_path = temp_dir_path.Append(L"gpu-watchdog.tmp");
+      // Please note that multiple instances of GPU process may reuse the same
+      // temporary file. That's OK because the file is written to only when
+      // the watchdog gets triggered and about to crash the process. The file
+      // should be deleted when the last handle is closed.
+      temp_file.Initialize(

[jbauman, 2016/05/17 22:57:20]

We're not doing anything special with the sandbox on non-windows, so we need to
make sure that this works there.

+          temp_file_path,
+          base::File::FLAG_OPEN_ALWAYS | base::File::FLAG_WRITE |
+              base::File::FLAG_DELETE_ON_CLOSE | base::File::FLAG_SHARE_DELETE);
+      if (temp_file.IsValid()) {
+        // Write a few bytes and wait for the write to flush.
+        const char temp_data[32] = {0};
+        base::ElapsedTimer timer;
+        temp_file.Write(0, temp_data, sizeof(temp_data));
+        temp_file.Flush();
+        io_check_duration_ = timer.Elapsed();
+
+        // Reset the flag and re-post this task to actually terminate unless
+        // acknowledge arrives in between.
+        use_temp_file_for_io_checking_ = false;
+        message_loop()->PostTask(
+            FROM_HERE,
+            base::Bind(
+                &GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang,
+                weak_factory_.GetWeakPtr()));
+        return;
+      }
+    }
+  }
+
 #if defined(USE_X11)
   XWindowAttributes attributes;
   XGetWindowAttributes(display_, window_, &attributes);