Convert AVDAs thread hang detection to be timer based

media/gpu/android_video_decode_accelerator.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/gpu/android_video_decode_accelerator.h"
 
 #include <stddef.h>
 
 #include <memory>
 
 #include "base/android/build_info.h"
 #include "base/auto_reset.h"
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
 #include "base/command_line.h"
 #include "base/lazy_instance.h"
 #include "base/logging.h"
+#include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_checker.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
 #include "gpu/command_buffer/service/gles2_cmd_decoder.h"
 #include "gpu/command_buffer/service/mailbox_manager.h"
 #include "gpu/ipc/service/gpu_channel.h"
 #include "media/base/android/media_codec_bridge.h"
 #include "media/base/android/media_codec_util.h"
@@ skipping 69 matching lines @@
 
 constexpr base::TimeDelta NoWaitTimeOut = base::TimeDelta::FromMicroseconds(0);
 
 constexpr base::TimeDelta IdleTimerTimeOut = base::TimeDelta::FromSeconds(1);
 
 // Time between when we notice an error, and when we actually notify somebody.
 // This is to prevent codec errors caused by SurfaceView fullscreen transitions
 // from breaking the pipeline, if we're about to be reset anyway.
 constexpr base::TimeDelta ErrorPostingDelay = base::TimeDelta::FromSeconds(2);
 
+constexpr base::TimeDelta kHungTaskDetectionTimeout =
+    base::TimeDelta::FromSeconds(2);
+
 // Maximum number of concurrent, incomplete codec creations that we'll allow
 // before turning off autodection of codec type.
 enum { kMaxConcurrentCodecAutodetections = 4 };
 
 // For RecordFormatChangedMetric.
 enum FormatChangedValue {
   CodecInitialized = false,
   MissingFormatChanged = true
 };
 
@@ skipping 59 matching lines @@
 //  - Starting and stopping a shared "construction" thread for instantiating and
 //    releasing MediaCodecs.
 //  - Tracking the number of outstanding tasks running on the construction
 //    thread. (For detecting when one of those tasks has hung indefinitely.)
 //  - Running a RepeatingTimer so that AVDAs can get a regular callback to
 //    DoIOTask().
 //  - Tracking the allocation of surfaces to AVDAs and delivering callbacks when
 //    surfaces are released.
 class AVDAManager {
  public:
+  class HangDetector : public base::MessageLoop::TaskObserver {
+   public:
+    HangDetector() {}
+
+    void WillProcessTask(const base::PendingTask& pending_task) override {
+      base::AutoLock l(lock_);
+      task_start_time_ = base::TimeTicks::Now();
+    }
+
+    void DidProcessTask(const base::PendingTask& pending_task) override {
+      base::AutoLock l(lock_);
+      task_start_time_.reset();
+    }
+
+    void Reset() {
+      base::AutoLock l(lock_);
+      task_start_time_.reset();
+    }
+
+    bool ThreadLikelyHung() {
+      base::AutoLock l(lock_);
+      if (task_start_time_.has_value()) {

[DaleCurtis, 2016/08/17 04:58:45]

is_null() is sufficient, ticks never starts at zero in practice.

+        return (base::TimeTicks::Now() - task_start_time_.value()) >
+               kHungTaskDetectionTimeout;
+      }
+      return false;
+    }
+
+   private:
+    base::Lock lock_;
+    base::Optional<base::TimeTicks> task_start_time_;
+
+    DISALLOW_COPY_AND_ASSIGN(HangDetector);
+  };
+
   // Make sure that the construction thread is started for |avda|.
   bool StartThread(AndroidVideoDecodeAccelerator* avda) {
     DCHECK(thread_checker_.CalledOnValidThread());
 
     // If we chose not to shut it down due to pending codec constructions, then
     // the thread might already be started even if there are no avda instances.
     // Plus, sometimes we just fail to start the thread.
     if (!construction_thread_.IsRunning()) {
       if (!construction_thread_.Start()) {
         LOG(ERROR) << "Failed to start construction thread.";
         return false;
       }
+      construction_thread_.task_runner()->PostTask(
+          FROM_HERE,
+          base::Bind(&base::MessageLoop::AddTaskObserver,
+                     base::Unretained(construction_thread_.message_loop()),
+                     &hang_detector_));
     }
 
     thread_avda_instances_.insert(avda);
     UMA_HISTOGRAM_ENUMERATION("Media.AVDA.NumAVDAInstances",
                               thread_avda_instances_.size(),
                               31);  // PRESUBMIT_IGNORE_UMA_MAX
     return true;
   }
 
   // |avda| will no longer need the construction thread.  Stop the thread if
   // this is the last instance.
   void StopThread(AndroidVideoDecodeAccelerator* avda) {
     DCHECK(thread_checker_.CalledOnValidThread());
 
     thread_avda_instances_.erase(avda);

[liberato (no reviews please), 2016/08/17 15:15:42]

if we end up not resetting below, then i think that |thread_avda_instances_| can
go away entirely, along with StopThread.

     if (!thread_avda_instances_.empty())
       return;
 
-    // Don't stop the thread if there are outstanding requests, since they
-    // might be hung.  They also might simply be incomplete, and the thread
-    // will stay running until we try to shut it down again.
-    base::AutoLock auto_lock(autodetection_info_.lock_);
-    if (autodetection_info_.outstanding_)
-      return;
-
-    construction_thread_.Stop();
+    // TODO: post a task to stop the thread.

[liberato (no reviews please), 2016/08/17 15:15:42]

not sure that posting a task to stop the thread is all that simple.  seems like
it'd be hard to guarantee that the thread is re-started properly if the shutdown
is racing with a future post that believes that it's already started.

+    hang_detector_.Reset();

[liberato (no reviews please), 2016/08/17 15:15:42]

i don't think that it should reset here.  for example, if we destroy the only
avda instance while an async codec config is hung, then we don't want to start
posting tasks again.  if it does eventually complete, it'll reset itself
normally even if the "AndReply" part cancels due to the invalidated weak ptr.

   }
 
   // Request periodic callback of |avda|->DoIOTask(). Does nothing if the
   // instance is already registered and the timer started. The first request
   // will start the repeating timer on an interval of DecodePollDelay.
   void StartTimer(AndroidVideoDecodeAccelerator* avda) {
     DCHECK(thread_checker_.CalledOnValidThread());
 
     timer_avda_instances_.insert(avda);
 
@@ skipping 22 matching lines @@
     timer_avda_instances_.erase(avda);
     if (timer_avda_instances_.empty())
       io_timer_.Stop();
   }
 
   scoped_refptr<base::SingleThreadTaskRunner> ConstructionTaskRunner() {
     DCHECK(thread_checker_.CalledOnValidThread());
     return construction_thread_.task_runner();
   }
 
-  // Called on the main thread when the construction thread will be doing work
-  // that can potentially hang (e.g., autodetection).  There may be several
-  // calls to this before any call to DoneUsingConstructionThread.
-  // Note that this should only be called from the main thread, else it's a race
-  // with IsCodecAutodetectionProbablySafe.
-  void StartUsingConstructionThread() {
-    DCHECK(thread_checker_.CalledOnValidThread());
-    base::AutoLock auto_lock(autodetection_info_.lock_);
-    ++autodetection_info_.outstanding_;
-  }
-
-  // Called on any thread after the potentially dangerous construction thread
-  // work completes safely.  May be called on any thread, including the
-  // construction thread.
-  // This assumes that requests are ordered, so please don't mix sync and async
-  // codec construction here.
-  void DoneUsingConstructionThread() {
-    base::AutoLock auto_lock_l(autodetection_info_.lock_);
-    DCHECK_GT(autodetection_info_.outstanding_, 0);
-    --autodetection_info_.outstanding_;
-  }
-
   // Return a hint about whether autodetecting the codec type is safe or not.
   bool IsCodecAutodetectionProbablySafe() {
-    base::AutoLock auto_lock_l(autodetection_info_.lock_);
-
-    return autodetection_info_.outstanding_ < kMaxConcurrentCodecAutodetections;
+    return !hang_detector_.ThreadLikelyHung();
   }
 
   // |avda| would like to use |surface_id|.  If it is not busy, then mark it
   // as busy and return true.  If it is busy, then replace any existing waiter,
   // make |avda| the current waiter, and return false.  Any existing waiter
   // is assumed to be on the way out, so we fail its allocation request.
   bool AllocateSurface(int surface_id, AndroidVideoDecodeAccelerator* avda) {
     // Nobody has to wait for no surface.
     if (surface_id == AndroidVideoDecodeAccelerator::Config::kNoSurfaceID)
       return true;
@@ skipping 87 matching lines @@
   SurfaceWaiterMap surface_waiter_map_;
 
   // Since we can't delete while iterating when using a set, defer erasure until
   // after iteration complete.
   bool timer_running_ = false;
   std::set<AndroidVideoDecodeAccelerator*> pending_erase_;
 
   // Repeating timer responsible for draining pending IO to the codecs.
   base::RepeatingTimer io_timer_;
 
-  // Data for determining if codec creation is hanging.
-  struct {
-    // Lock that protects other members of this struct.
-    base::Lock lock_;
-
-    // Number of currently pending work items of the construction thread.
-    int outstanding_ = 0;
-  } autodetection_info_;
-
   base::Thread construction_thread_;
+  HangDetector hang_detector_;
 
   base::ThreadChecker thread_checker_;
 
   DISALLOW_COPY_AND_ASSIGN(AVDAManager);
 };
 
 static base::LazyInstance<AVDAManager>::Leaky g_avda_manager =
     LAZY_INSTANCE_INITIALIZER;
 
 AndroidVideoDecodeAccelerator::CodecConfig::CodecConfig() {}
@@ skipping 682 matching lines @@
       g_avda_manager.Get().IsCodecAutodetectionProbablySafe();
 
   // If autodetection is disallowed, fall back to Chrome's software decoders
   // instead of using the software decoders provided by MediaCodec.
   if (!codec_config_->allow_autodetection_ &&
       IsMediaCodecSoftwareDecodingForbidden()) {
     OnCodecConfigured(nullptr);
     return;
   }
 
-  codec_config_->notify_completion_ = codec_config_->allow_autodetection_;
-  if (codec_config_->allow_autodetection_)
-    g_avda_manager.Get().StartUsingConstructionThread();
-
   // If we're not trying autodetection, then use the main thread.  The original
   // might be blocked.
   scoped_refptr<base::SingleThreadTaskRunner> task_runner =
       codec_config_->allow_autodetection_
           ? g_avda_manager.Get().ConstructionTaskRunner()
           : base::ThreadTaskRunnerHandle::Get();
   CHECK(task_runner);
 
   base::PostTaskAndReplyWithResult(
       task_runner.get(), FROM_HERE,
       base::Bind(&AndroidVideoDecodeAccelerator::ConfigureMediaCodecOnAnyThread,
                  codec_config_),
       base::Bind(&AndroidVideoDecodeAccelerator::OnCodecConfigured,
                  weak_this_factory_.GetWeakPtr()));
 }
 
 bool AndroidVideoDecodeAccelerator::ConfigureMediaCodecSynchronously() {
   state_ = WAITING_FOR_CODEC;
 
   // Decide whether to allow autodetection or not.  Since we're on the main
   // thread, and this request is unordered with respect to pending async config
   // attempts, don't record it.  It may break book-keeping, and there's not
   // much we can do anyway.
   codec_config_->allow_autodetection_ =
       g_avda_manager.Get().IsCodecAutodetectionProbablySafe();
-  codec_config_->notify_completion_ = false;
 
   ReleaseMediaCodec();
   std::unique_ptr<VideoCodecBridge> media_codec =
       ConfigureMediaCodecOnAnyThread(codec_config_);
   OnCodecConfigured(std::move(media_codec));
   return !!media_codec_;
 }
 
 std::unique_ptr<VideoCodecBridge>
 AndroidVideoDecodeAccelerator::ConfigureMediaCodecOnAnyThread(
     scoped_refptr<CodecConfig> codec_config) {
   TRACE_EVENT0("media", "AVDA::ConfigureMediaCodec");
 
   jobject media_crypto = codec_config->media_crypto_
                              ? codec_config->media_crypto_->obj()
                              : nullptr;
 
   // |needs_protected_surface_| implies encrypted stream.
   DCHECK(!codec_config->needs_protected_surface_ || media_crypto);
 
   const bool require_software_codec = !codec_config->allow_autodetection_;
 
   std::unique_ptr<VideoCodecBridge> codec(VideoCodecBridge::CreateDecoder(
       codec_config->codec_, codec_config->needs_protected_surface_,
       codec_config->initial_expected_coded_size_,
       codec_config->surface_.j_surface().obj(), media_crypto, true,
       require_software_codec));
 
-  // If we successfully completed after an autodetect, then let the other
-  // instances know that we didn't get stuck.
-  if (codec_config->notify_completion_)
-    g_avda_manager.Get().DoneUsingConstructionThread();
-
   return codec;
 }
 
 void AndroidVideoDecodeAccelerator::OnCodecConfigured(
     std::unique_ptr<VideoCodecBridge> media_codec) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(state_ == WAITING_FOR_CODEC || state_ == SURFACE_DESTROYED);
 
   // Record one instance of the codec being initialized.
   RecordFormatChangedMetric(FormatChangedValue::CodecInitialized);
@@ skipping 490 matching lines @@
   // backed by hardware, else it may hang too.  Post it to the construction
   // thread, and it'll get freed if things start working.  If things are
   // already working, then it'll be freed soon.
   //
   // We require software codecs when |allow_autodetection_| is false, so use
   // the stored value as a proxy for whether the MediaCodec is software backed
   // or not.
   if (!codec_config_->allow_autodetection_) {
     media_codec_.reset();
   } else {
-    g_avda_manager.Get().StartUsingConstructionThread();
-    scoped_refptr<base::SingleThreadTaskRunner> task_runner =
-        g_avda_manager.Get().ConstructionTaskRunner();
-    task_runner->DeleteSoon(FROM_HERE, media_codec_.release());
-    task_runner->PostTask(
-        FROM_HERE, base::Bind(&AVDAManager::DoneUsingConstructionThread,
-                              base::Unretained(g_avda_manager.Pointer())));
+    g_avda_manager.Get().ConstructionTaskRunner()->DeleteSoon(
+        FROM_HERE, media_codec_.release());
   }
 }
 
 // static
 bool AndroidVideoDecodeAccelerator::UseDeferredRenderingStrategy(
     const gpu::GpuPreferences& gpu_preferences) {
   return true;
 }
 
 // static
@@ skipping 78 matching lines @@
 
 bool AndroidVideoDecodeAccelerator::IsMediaCodecSoftwareDecodingForbidden()
     const {
   // Prevent MediaCodec from using its internal software decoders when we have
   // more secure and up to date versions in the renderer process.
   return !config_.is_encrypted && (codec_config_->codec_ == media::kCodecVP8 ||
                                    codec_config_->codec_ == media::kCodecVP9);
 }
 
 }  // namespace media