Detect configuration changes in H.264 video streams in the DXVA decoder

content/common/gpu/media/dxva_video_decode_accelerator_win.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/common/gpu/media/dxva_video_decode_accelerator_win.h"
 
 #if !defined(OS_WIN)
 #error This file should only be built on Windows.
 #endif   // !defined(OS_WIN)
 
@@ skipping 16 matching lines @@
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/shared_memory.h"
 #include "base/message_loop/message_loop.h"
 #include "base/path_service.h"
 #include "base/trace_event/trace_event.h"
 #include "base/win/windows_version.h"
 #include "build/build_config.h"
 #include "media/base/win/mf_initializer.h"
+#include "media/filters/h264_parser.h"
 #include "media/video/video_decode_accelerator.h"
 #include "third_party/angle/include/EGL/egl.h"
 #include "third_party/angle/include/EGL/eglext.h"
 #include "ui/gl/gl_bindings.h"
 #include "ui/gl/gl_context.h"
 #include "ui/gl/gl_fence.h"
 #include "ui/gl/gl_surface_egl.h"
 
 namespace {
 
@@ skipping 57 matching lines @@
 #endif
 
 // MF_XVP_PLAYBACK_MODE
 // Data type: UINT32 (treat as BOOL)
 // If this attribute is TRUE, the video processor will run in playback mode
 // where it allows callers to allocate output samples and allows last frame
 // regeneration (repaint).
 DEFINE_GUID(MF_XVP_PLAYBACK_MODE, 0x3c5d293f, 0xad67, 0x4e29, 0xaf, 0x12,
             0xcf, 0x3e, 0x23, 0x8a, 0xcc, 0xe9);
 
+// Provides scoped access to the underlying buffer in an IMFMediaBuffer
+// instance.
+class MediaBufferScopedPointer {
+ public:
+  MediaBufferScopedPointer(IMFMediaBuffer* media_buffer)
+      : media_buffer_(media_buffer),
+        buffer_(nullptr),
+        max_length_(0),
+        current_length_(0) {
+    HRESULT hr = media_buffer_->Lock(&buffer_, &max_length_, &current_length_);
+    CHECK(SUCCEEDED(hr));
+  }
+
+  ~MediaBufferScopedPointer() {
+    HRESULT hr = media_buffer_->Unlock();
+    CHECK(SUCCEEDED(hr));
+  }
+
+  uint8_t* get() {
+    return buffer_;
+  }
+
+  DWORD current_length() const {
+    return current_length_;
+  }
+
+ private:
+  base::win::ScopedComPtr<IMFMediaBuffer> media_buffer_;
+  uint8_t* buffer_;
+  DWORD max_length_;
+  DWORD current_length_;
+
+  DISALLOW_COPY_AND_ASSIGN(MediaBufferScopedPointer);
+};
+
 }  // namespace
 
 namespace content {
 
 static const media::VideoCodecProfile kSupportedProfiles[] = {
   media::H264PROFILE_BASELINE,
   media::H264PROFILE_MAIN,
   media::H264PROFILE_HIGH,
   media::VP8PROFILE_ANY,
   media::VP9PROFILE_ANY
@@ skipping 200 matching lines @@
       reinterpret_cast<EGLDeviceEXT>(egl_device), object_type, &device),
       "The eglQueryDeviceAttribEXT function failed to get the device",
       device_object);
 
   RETURN_ON_FAILURE(device, "Failed to get the ANGLE device", device_object);
 
   device_object = reinterpret_cast<T*>(device);
   return device_object;
 }
 
+H264ConfigChangeDetector::H264ConfigChangeDetector()
+    : last_sps_id_(0),
+      last_pps_id_(0),
+      config_changed_(false) {
+}
+
+H264ConfigChangeDetector::~H264ConfigChangeDetector() {
+}
+
+bool H264ConfigChangeDetector::DetectConfig(const uint8_t* stream,
+                                            unsigned int size) {
+  std::vector<uint8_t> sps;
+  std::vector<uint8_t> pps;
+  media::H264NALU nalu;
+  bool idr_seen = false;
+
+  media::H264Parser parser;
+  parser.SetStream(stream, size);
+  config_changed_ = false;
+
+  while (true) {
+    media::H264Parser::Result result = parser.AdvanceToNextNALU(&nalu);
+
+    if (result == media::H264Parser::kEOStream)
+      break;
+
+    if (result == media::H264Parser::kUnsupportedStream) {
+      DLOG(ERROR) << "Unsupported H.264 stream";
+      return false;
+    }
+
+    if (result != media::H264Parser::kOk) {
+      DLOG(ERROR) << "Failed to parse H.264 stream";
+      return false;
+    }
+
+    switch (nalu.nal_unit_type) {
+      case media::H264NALU::kSPS:
+        result = parser.ParseSPS(&last_sps_id_);
+        if (result == media::H264Parser::kUnsupportedStream) {
+          DLOG(ERROR) << "Unsupported SPS";
+          return false;
+        }
+
+        if (result != media::H264Parser::kOk) {
+          DLOG(ERROR) << "Could not parse SPS";
+          return false;
+        }
+
+        sps.assign(nalu.data, nalu.data + nalu.size);
+        break;
+
+      case media::H264NALU::kPPS:
+        result = parser.ParsePPS(&last_pps_id_);
+        if (result == media::H264Parser::kUnsupportedStream) {
+          DLOG(ERROR) << "Unsupported PPS";
+          return false;
+        }
+        if (result != media::H264Parser::kOk) {
+          DLOG(ERROR) << "Could not parse PPS";
+          return false;
+        }
+        pps.assign(nalu.data, nalu.data + nalu.size);
+        break;
+
+      case media::H264NALU::kIDRSlice:
+        idr_seen = true;
+        break;
+
+      default:
+        break;
+    }
+  }
+
+  if (!sps.empty() && sps != last_sps_) {
+    if (!last_sps_.empty() && idr_seen)

[sandersd (OOO until July 31), 2016/03/18 23:38:29]

I worry that Flash may send the SPS/PPS on one buffer, then the IDR slice in the
next buffer. Perhaps better would be |pending_config_changed_|, and then promote
that to |config_changed_| on IDR?

(That's a bit of a rabbit hole, since presumably you would also want a
pending_sps_ and pending_pps_ so that if it 'changes back' you could drop the
bit. I wouldn't worry about that for now.)

[ananta, 2016/03/19 00:00:59]

Thanks for pointing that out. Updated

+      config_changed_ = true;
+    last_sps_.swap(sps);
+  }
+
+  if (!pps.empty() && pps != last_pps_) {
+    if (!last_pps_.empty() && idr_seen)
+      config_changed_ = true;
+    last_pps_.swap(pps);
+  }
+  return true;
+}
 
 // Maintains information about a DXVA picture buffer, i.e. whether it is
 // available for rendering, the texture information, etc.
 struct DXVAVideoDecodeAccelerator::DXVAPictureBuffer {
  public:
   static linked_ptr<DXVAPictureBuffer> Create(
       const DXVAVideoDecodeAccelerator& decoder,
       const media::PictureBuffer& buffer,
       EGLConfig egl_config);
   ~DXVAPictureBuffer();
@@ skipping 437 matching lines @@
   RETURN_AND_NOTIFY_ON_FAILURE(
       SendMFTMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0),
       "Send MFT_MESSAGE_NOTIFY_BEGIN_STREAMING notification failed",
       PLATFORM_FAILURE, false);
 
   RETURN_AND_NOTIFY_ON_FAILURE(
       SendMFTMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0),
       "Send MFT_MESSAGE_NOTIFY_START_OF_STREAM notification failed",
       PLATFORM_FAILURE, false);
 
+  config_ = config;
+
   SetState(kNormal);
 
   StartDecoderThread();
   return true;
 }
 
 bool DXVAVideoDecodeAccelerator::CreateD3DDevManager() {
   TRACE_EVENT0("gpu", "DXVAVideoDecodeAccelerator_CreateD3DDevManager");
 
   HRESULT hr = E_FAIL;
@@ skipping 232 matching lines @@
     return;
 
   OutputBuffers::iterator it = output_picture_buffers_.find(picture_buffer_id);
   // If we didn't find the picture id in the |output_picture_buffers_| map we
   // try the |stale_output_picture_buffers_| map, as this may have been an
   // output picture buffer from before a resolution change, that at resolution
   // change time had yet to be displayed. The client is calling us back to tell
   // us that we can now recycle this picture buffer, so if we were waiting to
   // dispose of it we now can.
   if (it == output_picture_buffers_.end()) {
-    it = stale_output_picture_buffers_.find(picture_buffer_id);
-    RETURN_AND_NOTIFY_ON_FAILURE(it != stale_output_picture_buffers_.end(),
-        "Invalid picture id: " << picture_buffer_id, INVALID_ARGUMENT,);
-    main_thread_task_runner_->PostTask(
-        FROM_HERE,
-        base::Bind(&DXVAVideoDecodeAccelerator::DeferredDismissStaleBuffer,
-                   weak_this_factory_.GetWeakPtr(), picture_buffer_id));
+    if (!stale_output_picture_buffers_.empty()) {
+      it = stale_output_picture_buffers_.find(picture_buffer_id);
+      RETURN_AND_NOTIFY_ON_FAILURE(it != stale_output_picture_buffers_.end(),
+          "Invalid picture id: " << picture_buffer_id, INVALID_ARGUMENT,);
+      main_thread_task_runner_->PostTask(
+          FROM_HERE,
+          base::Bind(&DXVAVideoDecodeAccelerator::DeferredDismissStaleBuffer,
+                     weak_this_factory_.GetWeakPtr(), picture_buffer_id));
+    }
     return;
   }
 
   if (it->second->available() || it->second->waiting_to_reuse())
     return;
 
   if (use_keyed_mutex_ || using_angle_device_) {
     RETURN_AND_NOTIFY_ON_FAILURE(it->second->ReusePictureBuffer(),
                                  "Failed to reuse picture buffer",
                                  PLATFORM_FAILURE, );
@@ skipping 612 matching lines @@
   if (GetState() == kUninitialized)
     return;
 
   decoder_thread_.Stop();
   weak_this_factory_.InvalidateWeakPtrs();
   output_picture_buffers_.clear();
   stale_output_picture_buffers_.clear();
   pending_output_samples_.clear();
   pending_input_buffers_.clear();
   decoder_.Release();
+  pictures_requested_ = false;
 
   if (use_dx11_) {
     if (video_format_converter_mft_.get()) {
       video_format_converter_mft_->ProcessMessage(
           MFT_MESSAGE_NOTIFY_END_STREAMING, 0);
       video_format_converter_mft_.Release();
     }
     d3d11_device_context_.Release();
     d3d11_device_.Release();
     d3d11_device_manager_.Release();
     d3d11_query_.Release();
     dx11_video_format_converter_media_type_needs_init_ = true;
+    multi_threaded_.Release();
   } else {
     d3d9_.Release();
     d3d9_device_ex_.Release();
     device_manager_.Release();
     query_.Release();
   }
 
   SetState(kUninitialized);
 }
 
@@ skipping 134 matching lines @@
   DCHECK(decoder_thread_task_runner_->BelongsToCurrentThread());
 
   if (GetState() == kUninitialized)
     return;
 
   if (OutputSamplesPresent() || !pending_input_buffers_.empty()) {
     pending_input_buffers_.push_back(sample);
     return;
   }
 
+  // Check if the resolution, bit rate, etc changed in the stream. If yes we
+  // reinitialize the decoder to ensure that the stream decodes correctly.
+  bool config_changed = false;
+
+  HRESULT hr = CheckConfigChanged(sample.get(), &config_changed);
+  RETURN_AND_NOTIFY_ON_HR_FAILURE(hr, "Failed to check video stream config",
+      PLATFORM_FAILURE,);
+
+  if (config_changed) {
+    main_thread_task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&DXVAVideoDecodeAccelerator::ConfigChanged,
+                   weak_this_factory_.GetWeakPtr(),
+                   config_,
+                   sample));
+    return;
+  }
+
   if (!inputs_before_decode_) {
     TRACE_EVENT_ASYNC_BEGIN0("gpu", "DXVAVideoDecodeAccelerator.Decoding",
                              this);
   }
   inputs_before_decode_++;
 
-  HRESULT hr = decoder_->ProcessInput(0, sample.get(), 0);
+  hr = decoder_->ProcessInput(0, sample.get(), 0);
   // As per msdn if the decoder returns MF_E_NOTACCEPTING then it means that it
   // has enough data to produce one or more output samples. In this case the
   // recommended options are to
   // 1. Generate new output by calling IMFTransform::ProcessOutput until it
   //    returns MF_E_TRANSFORM_NEED_MORE_INPUT.
   // 2. Flush the input data
   // We implement the first option, i.e to retrieve the output sample and then
   // process the input again. Failure in either of these steps is treated as a
   // decoder failure.
   if (hr == MF_E_NOTACCEPTING) {
@@ skipping 57 matching lines @@
                  input_buffer_id));
 }
 
 void DXVAVideoDecodeAccelerator::HandleResolutionChanged(int width,
                                                          int height) {
   dx11_video_format_converter_media_type_needs_init_ = true;
 
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&DXVAVideoDecodeAccelerator::DismissStaleBuffers,
-                 weak_this_factory_.GetWeakPtr()));
+                 weak_this_factory_.GetWeakPtr(), false));
 
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&DXVAVideoDecodeAccelerator::RequestPictureBuffers,
                  weak_this_factory_.GetWeakPtr(),
                  width,
                  height));
 }
 
-void DXVAVideoDecodeAccelerator::DismissStaleBuffers() {
+void DXVAVideoDecodeAccelerator::DismissStaleBuffers(bool force) {
   OutputBuffers::iterator index;
 
   for (index = output_picture_buffers_.begin();
        index != output_picture_buffers_.end();
        ++index) {
-    if (index->second->available()) {
+    if (force || index->second->available()) {
       DVLOG(1) << "Dismissing picture id: " << index->second->id();
       client_->DismissPictureBuffer(index->second->id());
     } else {
       // Move to |stale_output_picture_buffers_| for deferred deletion.
       stale_output_picture_buffers_.insert(
           std::make_pair(index->first, index->second));
     }
   }
 
   output_picture_buffers_.clear();
@@ skipping 488 matching lines @@
       }
       hr = transform->SetOutputType(0, media_type.get(), 0);  // No flags
       RETURN_ON_HR_FAILURE(hr, "Failed to set output type", false);
       return true;
     }
     media_type.Release();
   }
   return false;
 }
 
+HRESULT DXVAVideoDecodeAccelerator::CheckConfigChanged(
+    IMFSample* sample, bool* config_changed) {
+  if (codec_ != media::kCodecH264)
+    return S_FALSE;
+
+  base::win::ScopedComPtr<IMFMediaBuffer> buffer;
+  HRESULT hr = sample->GetBufferByIndex(0, buffer.Receive());
+  RETURN_ON_HR_FAILURE(hr, "Failed to get buffer from input sample", hr);
+
+  MediaBufferScopedPointer scoped_media_buffer(buffer.get());
+
+  if (!config_change_detector_.DetectConfig(
+          scoped_media_buffer.get(),
+          scoped_media_buffer.current_length())) {
+    RETURN_ON_HR_FAILURE(E_FAIL, "Failed to detect H.264 stream config",
+        E_FAIL);
+  }
+  *config_changed = config_change_detector_.config_changed();
+  return S_OK;
+}
+
+void DXVAVideoDecodeAccelerator::ConfigChanged(
+    const Config& config,
+    const base::win::ScopedComPtr<IMFSample>& input_sample) {
+  DCHECK(main_thread_task_runner_->BelongsToCurrentThread());
+  DismissStaleBuffers(true);
+  Invalidate();
+  Initialize(config_, client_);
+  decoder_thread_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&DXVAVideoDecodeAccelerator::DecodeInternal,
+                 base::Unretained(this), input_sample));
+}
+
 }  // namespace content