GPU Watchdog to check I/O before terminating GPU process

content/gpu/gpu_watchdog_thread.cc

Index: content/gpu/gpu_watchdog_thread.cc
diff --git a/content/gpu/gpu_watchdog_thread.cc b/content/gpu/gpu_watchdog_thread.cc
index dddb40945af57816ceb155bd932c7a3603cfe69d..13efe46887c2dee32589de0b81bd6771dfad5716 100644
--- a/content/gpu/gpu_watchdog_thread.cc
+++ b/content/gpu/gpu_watchdog_thread.cc
@@ -12,6 +12,7 @@
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/debug/alias.h"
+#include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/location.h"
 #include "base/macros.h"
@@ -19,6 +20,7 @@
 #include "base/process/process.h"
 #include "base/single_thread_task_runner.h"
 #include "base/threading/platform_thread.h"
+#include "base/timer/elapsed_timer.h"
 #include "build/build_config.h"
 #include "content/public/common/content_switches.h"
 #include "content/public/common/result_codes.h"
@@ -80,6 +82,26 @@ GpuWatchdogThread::GpuWatchdogThread(int timeout)
   SetupXServer();
 #endif
   watched_message_loop_->AddTaskObserver(&task_observer_);
+
+  // Create a temp file for checking whether I/O is a bottleneck.
+  // This code runs on the main GPU thread before the sandbox is lowered.
+  base::FilePath temp_dir_path;
+  if (GetTempDir(&temp_dir_path)) {
+    base::FilePath temp_file_path =
+        temp_dir_path.Append(FILE_PATH_LITERAL("gpu-watchdog.tmp"));

[Will Harris, 2016/05/19 10:25:01]

use a constant for "gpu-watchdog.tmp" as it's used in two places.

[stanisc, 2016/06/09 00:33:46]

Done.

+    // Please note that multiple instances of GPU process may reuse the same
+    // temporary file. That's OK because the file is written to only when
+    // the watchdog gets triggered and about to crash the process. The file
+    // should be deleted when the last handle is closed.
+    temp_file_for_io_checking_.Initialize(
+        temp_file_path, base::File::FLAG_OPEN_ALWAYS | base::File::FLAG_WRITE |
+                            base::File::FLAG_DELETE_ON_CLOSE |
+                            base::File::FLAG_SHARE_DELETE);

[Will Harris, 2016/05/19 10:25:01]

two chrome's running on the same machine as same user e.g. chrome canary and
chrome stable, will use the same file but the share permissions here don't allow
the file to be opened by both.

[stanisc, 2016/05/19 18:47:26]

It does work with two clients. I've tested that. The file gets deleted when the
last client exits. If you look at the implementation in file.h The "share write"
flag is added by default unless the "restrict write" flag is specified.

+    if (!temp_file_for_io_checking_.IsValid()) {
+      LOG(ERROR) << "Couldn't create " << temp_file_path.value().c_str()
+                 << ", error: " << temp_file_for_io_checking_.error_details();
+    }
+  }
 }
 
 void GpuWatchdogThread::PostAcknowledge() {
@@ -230,14 +252,12 @@ void GpuWatchdogThread::OnCheck(bool after_suspend) {
   // Post a task to the watchdog thread to exit if the monitored thread does
   // not respond in time.
   task_runner()->PostDelayedTask(
-      FROM_HERE,
-      base::Bind(&GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang,
-                 weak_factory_.GetWeakPtr()),
+      FROM_HERE, base::Bind(&GpuWatchdogThread::BeginTerminating,
+                            weak_factory_.GetWeakPtr()),
       timeout);
 }
 
-// Use the --disable-gpu-watchdog command line switch to disable this.
-void GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang() {
+void GpuWatchdogThread::BeginTerminating() {
   // Should not get here while the system is suspended.
   DCHECK(!suspended_);
 
@@ -248,15 +268,38 @@ void GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang() {
   base::TimeDelta time_since_arm = current_cpu_time - arm_cpu_time_;
   if (use_thread_cpu_time_ && (time_since_arm < timeout_)) {
     message_loop()->PostDelayedTask(
-        FROM_HERE,
-        base::Bind(
-            &GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang,
-            weak_factory_.GetWeakPtr()),
+        FROM_HERE, base::Bind(&GpuWatchdogThread::BeginTerminating,
+                              weak_factory_.GetWeakPtr()),
         timeout_ - time_since_arm);
     return;
   }
 #endif
 
+  // If the machine is busy with heavy I/O activity this should defer
+  // termination until the I/O queue clears (on the drive that contains
+  // the temp directory).
+  if (temp_file_for_io_checking_.IsValid()) {
+    // Write a few bytes and wait for the write to flush.
+    const char temp_data[32] = {0};
+    base::ElapsedTimer timer;
+    temp_file_for_io_checking_.Write(0, temp_data, sizeof(temp_data));
+    temp_file_for_io_checking_.Flush();
+    io_check_duration_ = timer.Elapsed();
+  }
+
+  // Post a new task to actually terminate unless an acknowledge from the
+  // watched thread arrives in between.
+  message_loop()->PostTask(
+      FROM_HERE,
+      base::Bind(&GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang,
+                 weak_factory_.GetWeakPtr()));
+}
+
+// Use the --disable-gpu-watchdog command line switch to disable this.
+void GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang() {
+  // Should not get here while the system is suspended.
+  DCHECK(!suspended_);
+
   // If the watchdog woke up significantly behind schedule, disarm and reset
   // the watchdog check. This is to prevent the watchdog thread from terminating
   // when a machine wakes up from sleep or hibernation, which would otherwise
@@ -348,9 +391,12 @@ void GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang() {
   // This is the time since the watchdog was armed, in 100ns intervals,
   // ignoring time where the computer is suspended.
   ULONGLONG interrupt_delay = fire_interrupt_time - arm_interrupt_time_;
-
   base::debug::Alias(&interrupt_delay);
+
+  base::ThreadTicks current_cpu_time = GetWatchedThreadTime();
   base::debug::Alias(&current_cpu_time);
+
+  base::TimeDelta time_since_arm = current_cpu_time - arm_cpu_time_;
   base::debug::Alias(&time_since_arm);
 
   bool using_thread_ticks = base::ThreadTicks::IsSupported();