Attempt decoder fallback if first decode fails

media/filters/decoder_stream.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/decoder_stream.h"
 
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
@@ skipping 41 matching lines @@
       state_(STATE_UNINITIALIZED),
       stream_(NULL),
       decoder_selector_(new DecoderSelector<StreamType>(task_runner,
                                                         std::move(decoders),
                                                         media_log)),
       decoded_frames_since_fallback_(0),
       active_splice_(false),
       decoding_eos_(false),
       pending_decode_requests_(0),
       duration_tracker_(8),
-      weak_factory_(this) {}
+      weak_factory_(this),
+      decode_weak_factory_(this) {}
 
 template <DemuxerStream::Type StreamType>
 DecoderStream<StreamType>::~DecoderStream() {
   FUNCTION_DVLOG(2);
   DCHECK(task_runner_->BelongsToCurrentThread());
 
   decoder_selector_.reset();
 
   if (!init_cb_.is_null()) {
     task_runner_->PostTask(FROM_HERE,
                            base::Bind(base::ResetAndReturn(&init_cb_), false));
   }
   if (!read_cb_.is_null()) {
     task_runner_->PostTask(FROM_HERE, base::Bind(
         base::ResetAndReturn(&read_cb_), ABORTED, scoped_refptr<Output>()));
   }
   if (!reset_cb_.is_null())
     task_runner_->PostTask(FROM_HERE, base::ResetAndReturn(&reset_cb_));
 
   stream_ = NULL;
+  pending_buffers_.clear();

[sandersd (OOO until July 31), 2016/04/14 23:10:30]

You don't actually need to do this, the destructor will handle it for you.

[tguilbert, 2016/04/15 00:29:18]

Done.

+  fallback_buffers_.clear();
   decoder_.reset();
   decrypting_demuxer_stream_.reset();
 }
 
 template <DemuxerStream::Type StreamType>
 std::string DecoderStream<StreamType>::GetStreamTypeString() {
   return DecoderStreamTraits<StreamType>::ToString();
 }
 
 template <DemuxerStream::Type StreamType>
@@ skipping 143 matching lines @@
 }
 
 template <DemuxerStream::Type StreamType>
 void DecoderStream<StreamType>::OnDecoderSelected(
     scoped_ptr<Decoder> selected_decoder,
     scoped_ptr<DecryptingDemuxerStream> decrypting_demuxer_stream) {
   FUNCTION_DVLOG(2) << ": "
                     << (selected_decoder ? selected_decoder->GetDisplayName()
                                          : "No decoder selected.");
   DCHECK(task_runner_->BelongsToCurrentThread());
-  DCHECK(state_ == STATE_INITIALIZING || state_ == STATE_REINITIALIZING_DECODER)
+  DCHECK(state_ == STATE_INITIALIZING ||
+         state_ == STATE_REINITIALIZING_DECODER ||
+         state_ == STATE_CONFIG_CHANGE_RECEIVED_WHILE_REINITIALIZING_DECODER)
       << state_;
   if (state_ == STATE_INITIALIZING) {
     DCHECK(!init_cb_.is_null());
     DCHECK(read_cb_.is_null());
     DCHECK(reset_cb_.is_null());
+  } else if (state_ == STATE_REINITIALIZING_DECODER) {
+    DCHECK(decoder_);
+  }
+
+  // TODO(tguilbert): crbug.com/603713 support config changes on decoder reinit.
+  if (state_ == STATE_CONFIG_CHANGE_RECEIVED_WHILE_REINITIALIZING_DECODER) {
+    CompleteDecoderReinitialization(false);

[tguilbert, 2016/04/14 22:26:42]

Sorry, I didn't take into account what should be done with |selected_decoder|.
Looking into this now.

[tguilbert, 2016/04/15 00:29:18]

Done.

+    return;
+  }
+
+  // Push the pending buffers to the front of the fallback buffers
+  if (fallback_buffers_.empty()) {
+    fallback_buffers_.swap(pending_buffers_);
   } else {
-    DCHECK(decoder_);
+    fallback_buffers_.insert(fallback_buffers_.begin(),
+                             pending_buffers_.begin(), pending_buffers_.end());
+    pending_buffers_.clear();
   }
 
   previous_decoder_ = std::move(decoder_);
   decoded_frames_since_fallback_ = 0;
   decoder_ = std::move(selected_decoder);
   if (decrypting_demuxer_stream) {
     decrypting_demuxer_stream_ = std::move(decrypting_demuxer_stream);
     stream_ = decrypting_demuxer_stream_.get();
   }
 
@@ skipping 48 matching lines @@
   TRACE_EVENT_ASYNC_BEGIN2(
       "media", GetTraceString<StreamType>(), this, "key frame",
       !buffer->end_of_stream() && buffer->is_key_frame(), "timestamp (ms)",
       !buffer->end_of_stream() ? buffer->timestamp().InMilliseconds() : 0);
 
   if (buffer->end_of_stream())
     decoding_eos_ = true;
   else if (buffer->duration() != kNoTimestamp())
     duration_tracker_.AddSample(buffer->duration());
 
+  // Don't save the buffer if we have successfully decoded a frame or if it's
+  // the EOS buffer we send to flush the decoder.

[sandersd (OOO until July 31), 2016/04/14 23:10:30]

I still think this would be more clear if ReadFromDemuxerStream() called a
shared helper which would never push buffers into |pending_buffers_|. Right now
it's not trivial to understand the exact usage of the two lists, nor the
conditions under which they should be updated.

[tguilbert, 2016/04/15 00:29:18]

Done. After our offline discussion, I realized you probably meant FlushDecoder()
instead of ReadFromDemuxerStream() (which was the source of my confusion).

+  if (!decoded_frames_since_fallback_ &&
+      !(state_ == STATE_FLUSHING_DECODER && buffer->end_of_stream())) {
+    pending_buffers_.push_back(buffer);
+  }
+
   ++pending_decode_requests_;
-  decoder_->Decode(buffer,
-                   base::Bind(&DecoderStream<StreamType>::OnDecodeDone,
-                              weak_factory_.GetWeakPtr(),
-                              buffer_size,
-                              buffer->end_of_stream()));
+  decoder_->Decode(buffer, base::Bind(&DecoderStream<StreamType>::OnDecodeDone,
+                                      decode_weak_factory_.GetWeakPtr(),
+                                      buffer_size, buffer->end_of_stream()));
 }
 
 template <DemuxerStream::Type StreamType>
 void DecoderStream<StreamType>::FlushDecoder() {
   Decode(DecoderBuffer::CreateEOSBuffer());
 }
 
 template <DemuxerStream::Type StreamType>
 void DecoderStream<StreamType>::OnDecodeDone(int buffer_size,
                                              bool end_of_stream,
                                              DecodeStatus status) {
   FUNCTION_DVLOG(2) << ": " << status;
+
   DCHECK(state_ == STATE_NORMAL || state_ == STATE_FLUSHING_DECODER ||
          state_ == STATE_PENDING_DEMUXER_READ || state_ == STATE_ERROR)
       << state_;
   DCHECK_GT(pending_decode_requests_, 0);
 
   --pending_decode_requests_;
 
   TRACE_EVENT_ASYNC_END0("media", GetTraceString<StreamType>(), this);
 
   if (end_of_stream) {
     DCHECK(!pending_decode_requests_);
     decoding_eos_ = false;
   }
 
   if (state_ == STATE_ERROR) {
     DCHECK(read_cb_.is_null());
     return;
   }
 
   // Drop decoding result if Reset() was called during decoding.
   // The resetting process will be handled when the decoder is reset.
   if (!reset_cb_.is_null())
     return;
 
   switch (status) {
     case DecodeStatus::DECODE_ERROR:
+      if (!decoded_frames_since_fallback_) {
+        pending_decode_requests_ = 0;
+        // Prevent all pending decode requests from being called back.

[sandersd (OOO until July 31), 2016/04/14 23:10:30]

Newline before comment.

[tguilbert, 2016/04/15 00:29:18]

Done.

+        decode_weak_factory_.InvalidateWeakPtrs();
+
+        state_ = STATE_REINITIALIZING_DECODER;
+        decoder_selector_->SelectDecoder(
+            stream_, nullptr,
+            base::Bind(&DecoderStream<StreamType>::OnDecoderSelected,
+                       weak_factory_.GetWeakPtr()),
+            base::Bind(&DecoderStream<StreamType>::OnDecodeOutputReady,
+                       weak_factory_.GetWeakPtr()),
+            waiting_for_decryption_key_cb_);
+        return;
+      }
       state_ = STATE_ERROR;
       MEDIA_LOG(ERROR, media_log_) << GetStreamTypeString() << " decode error";
       ready_outputs_.clear();
       if (!read_cb_.is_null())
         SatisfyRead(DECODE_ERROR, NULL);
       return;
 
     case DecodeStatus::ABORTED:
       // Decoder can return DecodeStatus::ABORTED during Reset() or during
       // destruction.
@@ skipping 35 matching lines @@
   if (state_ == STATE_ERROR) {
     DCHECK(read_cb_.is_null());
     return;
   }
 
   // Drop decoding result if Reset() was called during decoding.
   // The resetting process will be handled when the decoder is reset.
   if (!reset_cb_.is_null())
     return;
 
+  ++decoded_frames_since_fallback_;
+
+  if (!pending_buffers_.empty())

[sandersd (OOO until July 31), 2016/04/14 23:10:30]

No need for a condition, it can just be unconditionally cleared.

[tguilbert, 2016/04/15 00:29:18]

Oops! Artifact of the previous patchset.

+    pending_buffers_.clear();
+
   if (!read_cb_.is_null()) {
     // If |ready_outputs_| was non-empty, the read would have already been
     // satisifed by Read().
     DCHECK(ready_outputs_.empty());
     SatisfyRead(OK, output);
     return;
   }
 
   // Store decoded output.
   ready_outputs_.push_back(output);
 
   // Destruct any previous decoder once we've decoded enough frames to ensure
   // that it's no longer in use.
   if (previous_decoder_ &&
-      ++decoded_frames_since_fallback_ > limits::kMaxVideoFrames) {
+      decoded_frames_since_fallback_ > limits::kMaxVideoFrames) {
     previous_decoder_.reset();
   }
 }
 
 template <DemuxerStream::Type StreamType>
 void DecoderStream<StreamType>::ReadFromDemuxerStream() {
   FUNCTION_DVLOG(2);
   DCHECK_EQ(state_, STATE_NORMAL);
   DCHECK(CanDecodeMore());
   DCHECK(reset_cb_.is_null());
 
+  if (!fallback_buffers_.empty()) {
+    scoped_refptr<DecoderBuffer> buffer = fallback_buffers_.front();
+    fallback_buffers_.pop_front();
+    Decode(buffer);
+    return;
+  }
+
   state_ = STATE_PENDING_DEMUXER_READ;
   stream_->Read(base::Bind(&DecoderStream<StreamType>::OnBufferReady,
                            weak_factory_.GetWeakPtr()));
 }
 
 template <DemuxerStream::Type StreamType>
 void DecoderStream<StreamType>::OnBufferReady(
     DemuxerStream::Status status,
     const scoped_refptr<DecoderBuffer>& buffer) {
   FUNCTION_DVLOG(2) << ": " << status << ", "
                     << (buffer.get() ? buffer->AsHumanReadableString()
                                      : "NULL");
 
   DCHECK(task_runner_->BelongsToCurrentThread());
-  DCHECK(state_ == STATE_PENDING_DEMUXER_READ || state_ == STATE_ERROR)
-      << state_;
+  if (decoded_frames_since_fallback_) {
+    DCHECK(state_ == STATE_PENDING_DEMUXER_READ || state_ == STATE_ERROR)
+        << state_;
+  } else {
+    DCHECK(state_ == STATE_PENDING_DEMUXER_READ || state_ == STATE_ERROR ||
+           STATE_REINITIALIZING_DECODER)
+        << state_;
+  }
   DCHECK_EQ(buffer.get() != NULL, status == DemuxerStream::kOk) << status;
 
+  // If parallel decode requests are supported, multiple read requests might
+  // have been sent to the demuxer. The buffers might arrive while the decoder
+  // is reinitializing after falling back on first decode error.
+  if (state_ == STATE_REINITIALIZING_DECODER &&
+      !decoded_frames_since_fallback_) {
+    switch (status) {
+      case DemuxerStream::kOk:
+        // Save valid buffers to be consumed by the new decoder.
+        // |pending_buffers_| is copied to |fallback_buffers| in
+        // OnDecoderSelected().
+        pending_buffers_.push_back(buffer);
+        break;
+      case DemuxerStream::kConfigChanged:
+        // TODO(tguilbert): crbug.com/603713
+        // |decoder_| might have a stale config by the time it is reinitialized.
+        // Ideally, we would save the config from |stream_| and reinitialize the
+        // decoder by playing back the sequence of buffers and config changes.
+        state_ = STATE_CONFIG_CHANGE_RECEIVED_WHILE_REINITIALIZING_DECODER;
+        pending_buffers_.clear();
+        break;
+      case DemuxerStream::kAborted:
+        // |this| will read from the demuxer stream again in OnDecoderSelected()
+        // and receive a kAborted then.
+        pending_buffers_.clear();
+        break;
+      default:
+        NOTREACHED();
+        break;
+    }
+    return;
+  }
+
   // Decoding has been stopped (e.g due to an error).
   if (state_ != STATE_PENDING_DEMUXER_READ) {
     DCHECK(state_ == STATE_ERROR);
     DCHECK(read_cb_.is_null());
     return;
   }
 
   state_ = STATE_NORMAL;
 
   if (status == DemuxerStream::kConfigChanged) {
     FUNCTION_DVLOG(2) << ": " << "ConfigChanged";
     DCHECK(stream_->SupportsConfigChanges());
 
+    // Pending buffers might not match the reinitialiazed decoder's new config
+    pending_buffers_.clear();
+
     if (!config_change_observer_cb_.is_null())
       config_change_observer_cb_.Run();
 
     state_ = STATE_FLUSHING_DECODER;
     if (!reset_cb_.is_null()) {
       // If we are using DecryptingDemuxerStream, we already called DDS::Reset()
       // which will continue the resetting process in it's callback.
       if (!decrypting_demuxer_stream_)
         Reset(base::ResetAndReturn(&reset_cb_));
       // Reinitialization will continue after Reset() is done.
@@ skipping 131 matching lines @@
   }
 
   // The resetting process will be continued in OnDecoderReinitialized().
   ReinitializeDecoder();
 }
 
 template class DecoderStream<DemuxerStream::VIDEO>;
 template class DecoderStream<DemuxerStream::AUDIO>;
 
 }  // namespace media