GPU Watchdog to check I/O before terminating GPU process

content/gpu/gpu_watchdog_thread.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/gpu/gpu_watchdog_thread.h"
 
 #include <errno.h>
 #include <stdint.h>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/debug/alias.h"
+#include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/location.h"
 #include "base/macros.h"
 #include "base/power_monitor/power_monitor.h"
 #include "base/process/process.h"
 #include "base/single_thread_task_runner.h"
 #include "base/threading/platform_thread.h"
+#include "base/timer/elapsed_timer.h"
 #include "build/build_config.h"
 #include "content/public/common/content_switches.h"
 #include "content/public/common/result_codes.h"
 
 #if defined(OS_WIN)
 #include <windows.h>
 #endif
 
 namespace content {
 namespace {
@@ skipping 12 matching lines @@
       timeout_(base::TimeDelta::FromMilliseconds(timeout)),
       armed_(false),
       task_observer_(this),
       use_thread_cpu_time_(true),
       responsive_acknowledge_count_(0),
 #if defined(OS_WIN)
       watched_thread_handle_(0),
       arm_cpu_time_(),
 #endif
       suspended_(false),
+      use_temp_file_for_io_checking_(false),
 #if defined(USE_X11)
       display_(NULL),
       window_(0),
       atom_(None),
 #endif
       weak_factory_(this) {
   DCHECK(timeout >= 0);
 
 #if defined(OS_WIN)
   // GetCurrentThread returns a pseudo-handle that cannot be used by one thread
@@ skipping 137 matching lines @@
   // Do not create any new termination tasks if one has already been created
   // or the system is suspended.
   if (armed_ || suspended_)
     return;
 
   // Must set armed before posting the task. This task might be the only task
   // that will activate the TaskObserver on the watched thread and it must not
   // miss the false -> true transition.
   armed_ = true;
 
+  use_temp_file_for_io_checking_ = true;
+
 #if defined(OS_WIN)
   arm_cpu_time_ = GetWatchedThreadTime();
 
   QueryUnbiasedInterruptTime(&arm_interrupt_time_);
 #endif
 
   check_time_ = base::Time::Now();
   check_timeticks_ = base::TimeTicks::Now();
   // Immediately after the computer is woken up from being suspended it might
   // be pretty sluggish, so allow some extra time before the next timeout.
@@ skipping 39 matching lines @@
   // If the watchdog woke up significantly behind schedule, disarm and reset
   // the watchdog check. This is to prevent the watchdog thread from terminating
   // when a machine wakes up from sleep or hibernation, which would otherwise
   // appear to be a hang.
   if (base::Time::Now() > suspension_timeout_) {
     armed_ = false;
     OnCheck(true);
     return;
   }
 
+  // If the watchdog woke up while there is a heavy I/O we want to give the
+  // machine a chance to clear the I/O queue before terminating the process.
+  if (use_temp_file_for_io_checking_) {

[jbauman, 2016/05/17 22:57:20]

Instead of using a flag here, we should make this into a separate method.
Everything that currently does a PostTask of
GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang would instead do a
posttask of the new method (GpuWatchdogThread::CheckForIo ?), and this new
method would be the only thing to PostTask
GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang.

[stanisc, 2016/05/18 00:01:30]

Good point! I didn't realize that adding sandbox rules works only on Windows. I
might have to go back to the first iteration where I created the temp file
before lowering the sandbox (see patch set 1). Another option is to limit this
to Windows only.

+    base::File temp_file;
+    base::FilePath temp_dir_path;
+    if (GetTempDir(&temp_dir_path)) {
+      base::FilePath temp_file_path = temp_dir_path.Append(L"gpu-watchdog.tmp");
+      // Please note that multiple instances of GPU process may reuse the same
+      // temporary file. That's OK because the file is written to only when
+      // the watchdog gets triggered and about to crash the process. The file
+      // should be deleted when the last handle is closed.
+      temp_file.Initialize(

[jbauman, 2016/05/17 22:57:20]

We're not doing anything special with the sandbox on non-windows, so we need to
make sure that this works there.

+          temp_file_path,
+          base::File::FLAG_OPEN_ALWAYS | base::File::FLAG_WRITE |
+              base::File::FLAG_DELETE_ON_CLOSE | base::File::FLAG_SHARE_DELETE);
+      if (temp_file.IsValid()) {
+        // Write a few bytes and wait for the write to flush.
+        const char temp_data[32] = {0};
+        base::ElapsedTimer timer;
+        temp_file.Write(0, temp_data, sizeof(temp_data));
+        temp_file.Flush();
+        io_check_duration_ = timer.Elapsed();
+
+        // Reset the flag and re-post this task to actually terminate unless
+        // acknowledge arrives in between.
+        use_temp_file_for_io_checking_ = false;
+        message_loop()->PostTask(
+            FROM_HERE,
+            base::Bind(
+                &GpuWatchdogThread::DeliberatelyTerminateToRecoverFromHang,
+                weak_factory_.GetWeakPtr()));
+        return;
+      }
+    }
+  }
+
 #if defined(USE_X11)
   XWindowAttributes attributes;
   XGetWindowAttributes(display_, window_, &attributes);
 
   XSelectInput(display_, window_, PropertyChangeMask);
   SetupXChangeProp();
 
   XFlush(display_);
 
   // We wait for the property change event with a timeout. If it arrives we know
@@ skipping 177 matching lines @@
     // calls into kernel level repeatedly, giving up its quanta before it is
     // tracked, for example a loop that repeatedly Sleeps.
     return base::ThreadTicks() +
            base::TimeDelta::FromMilliseconds(static_cast<int64_t>(
                (user_time64.QuadPart + kernel_time64.QuadPart) / 10000));
   }
 }
 #endif
 
 }  // namespace content