Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(75)

Side by Side Diff: content/renderer/media/gpu/rtc_video_decoder.cc

Issue 2418613002: Proxy RtcVideoDecoder calls to a media::VideoDecoder.
Patch Set: Comments addressed and unittests updated. Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2013 The Chromium Authors. All rights reserved. 1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "content/renderer/media/gpu/rtc_video_decoder.h" 5 #include "content/renderer/media/gpu/rtc_video_decoder.h"
6 6
7 #include <utility> 7 #include <utility>
8 8
9 #include "base/bind.h" 9 #include "base/bind.h"
10 #include "base/logging.h" 10 #include "base/logging.h"
11 #include "base/memory/ref_counted.h" 11 #include "base/memory/ref_counted.h"
12 #include "base/metrics/histogram_macros.h" 12 #include "base/metrics/histogram_macros.h"
13 #include "base/numerics/safe_conversions.h" 13 #include "base/numerics/safe_conversions.h"
14 #include "base/stl_util.h"
14 #include "base/synchronization/waitable_event.h" 15 #include "base/synchronization/waitable_event.h"
15 #include "base/task_runner_util.h" 16 #include "base/task_runner_util.h"
16 #include "content/renderer/media/webrtc/webrtc_video_frame_adapter.h" 17 #include "content/renderer/media/webrtc/webrtc_video_frame_adapter.h"
17 #include "gpu/command_buffer/common/mailbox_holder.h" 18 #include "media/base/decoder_buffer.h"
18 #include "media/base/bind_to_current_loop.h" 19 #include "media/base/encryption_scheme.h"
19 #include "media/renderers/gpu_video_accelerator_factories.h" 20 #include "media/base/video_decoder.h"
20 #include "third_party/skia/include/core/SkBitmap.h"
21 #include "third_party/webrtc/base/bind.h" 21 #include "third_party/webrtc/base/bind.h"
22 #include "third_party/webrtc/base/refcount.h" 22 #include "third_party/webrtc/base/refcount.h"
23 #include "third_party/webrtc/modules/video_coding/codecs/h264/include/h264.h" 23 #include "third_party/webrtc/modules/video_coding/codecs/h264/include/h264.h"
24 #include "third_party/webrtc/video_frame.h" 24 #include "third_party/webrtc/video_frame.h"
25 25
26 #if defined(OS_WIN) 26 #if defined(OS_WIN)
27 #include "base/command_line.h" 27 #include "base/command_line.h"
28 #include "base/win/windows_version.h" 28 #include "base/win/windows_version.h"
29 #include "content/public/common/content_switches.h" 29 #include "content/public/common/content_switches.h"
30 #endif // defined(OS_WIN) 30 #endif // defined(OS_WIN)
31 31
32 namespace content { 32 namespace content {
33 namespace {
33 34
34 const int32_t RTCVideoDecoder::ID_LAST = 0x3FFFFFFF; 35 const int32_t ID_LAST = 0x3FFFFFFF;
35 const int32_t RTCVideoDecoder::ID_HALF = 0x20000000; 36 const int32_t ID_HALF = 0x20000000;
36 const int32_t RTCVideoDecoder::ID_INVALID = -1; 37 const int32_t ID_INVALID = -1;
37 38
38 // Number of consecutive frames that can be lost due to a VDA error before 39 // Number of consecutive frames that can be lost due to decoder errors before
39 // falling back to SW implementation. 40 // falling back to SW implementation.
40 const uint32_t kNumVDAErrorsBeforeSWFallback = 5; 41 const uint32_t kNumDecoderErrorsBeforeSWFallback = 5;
41 42
42 // Maximum number of concurrent VDA::Decode() operations RVD will maintain. 43 // Maximum number of pending buffers that are waiting to be sent to the remote
43 // Higher values allow better pipelining in the GPU, but also require more 44 // decoder.
44 // resources. 45 const size_t kMaxNumOfPendingBuffers = 8;
45 static const size_t kMaxInFlightDecodes = 8;
46 46
47 // Number of allocated shared memory segments. 47 // These values will be used to initialize the decoder before we know what the
48 static const size_t kNumSharedMemorySegments = 16; 48 // size of the video will be. These were chosen to match the default values in
49 // media/video/video_decode_accelerator.h.
50 const int kDefaultVideoWidth = 320;
51 const int kDefaultVideoHeight = 240;
49 52
50 // Maximum number of pending WebRTC buffers that are waiting for shared memory. 53 // Copy the data from |encoded_image|, creating a new media::DecoderBuffer.
51 static const size_t kMaxNumOfPendingBuffers = 8; 54 scoped_refptr<media::DecoderBuffer> CreateDecoderBuffer(
55 const webrtc::EncodedImage& encoded_image) {
56 auto decoder_buffer = media::DecoderBuffer::CopyFrom(encoded_image._buffer,
57 encoded_image._length);
58 decoder_buffer->set_timestamp(
59 base::TimeDelta::FromInternalValue(encoded_image._timeStamp));
60 return decoder_buffer;
61 }
52 62
53 RTCVideoDecoder::BufferData::BufferData(int32_t bitstream_buffer_id, 63 // Return a media::VideoDecoderConfig for the given WebRTC |codec_type| and
54 uint32_t timestamp, 64 // |size|. This object will be used to initialize the remote video decoder. If
55 size_t size, 65 // |codec_type| is not supported, returns an invalid config object.
56 const gfx::Rect& visible_rect) 66 media::VideoDecoderConfig GetVideoDecoderConfigForCodec(
57 : bitstream_buffer_id(bitstream_buffer_id), 67 webrtc::VideoCodecType codec_type,
58 timestamp(timestamp), 68 const gfx::Size& size) {
59 size(size), 69 media::VideoCodecProfile profile;
60 visible_rect(visible_rect) {} 70 media::VideoCodec codec;
71 switch (codec_type) {
72 case webrtc::kVideoCodecVP8:
73 profile = media::VP8PROFILE_ANY;
74 codec = media::kCodecVP8;
75 break;
76 case webrtc::kVideoCodecH264:
77 profile = media::H264PROFILE_MAIN;
78 codec = media::kCodecH264;
79 break;
80 case webrtc::kVideoCodecVP9:
81 profile = media::VP9PROFILE_PROFILE0;
82 codec = media::kCodecVP9;
83 break;
84 default:
85 return media::VideoDecoderConfig();
86 }
61 87
62 RTCVideoDecoder::BufferData::BufferData() {} 88 return media::VideoDecoderConfig(
89 codec, profile, media::PIXEL_FORMAT_ARGB, /* pixel_format */
90 media::COLOR_SPACE_UNSPECIFIED, /* color_space */
91 size, /* coded_size */
92 gfx::Rect(size), /* visible_rect */
93 size, /* natural_size */
94 std::vector<uint8_t>(), /* extra_data */
95 media::EncryptionScheme());
96 }
63 97
64 RTCVideoDecoder::BufferData::~BufferData() {} 98 bool IsBufferAfterReset(int32_t id_buffer, int32_t id_reset) {
99 DVLOG(2) << __func__;
100 if (id_reset == ID_INVALID)
101 return true;
102 int32_t diff = id_buffer - id_reset;
103 if (diff <= 0)
104 diff += ID_LAST + 1;
105 return diff < ID_HALF;
106 }
65 107
66 RTCVideoDecoder::RTCVideoDecoder(webrtc::VideoCodecType type, 108 bool IsFirstBufferAfterReset(int32_t id_buffer, int32_t id_reset) {
67 media::GpuVideoAcceleratorFactories* factories) 109 DVLOG(2) << __func__;
68 : vda_error_counter_(0), 110 if (id_reset == ID_INVALID)
111 return id_buffer == 0;
112 return id_buffer == ((id_reset + 1) & ID_LAST);
113 }
114
115 } // namespace
116
117 RTCVideoDecoder::RTCVideoDecoder(
118 webrtc::VideoCodecType type,
119 const CreateVideoDecoderCB& create_video_decoder_cb,
120 const scoped_refptr<base::SingleThreadTaskRunner>& decoder_task_runner)
121 : decoder_error_counter_(0),
69 video_codec_type_(type), 122 video_codec_type_(type),
70 factories_(factories), 123 create_video_decoder_cb_(create_video_decoder_cb),
71 decoder_texture_target_(0), 124 decoder_task_runner_(decoder_task_runner),
72 pixel_format_(media::PIXEL_FORMAT_UNKNOWN),
73 next_picture_buffer_id_(0),
74 state_(UNINITIALIZED), 125 state_(UNINITIALIZED),
75 decode_complete_callback_(nullptr), 126 decode_complete_callback_(nullptr),
76 num_shm_buffers_(0), 127 next_decoder_buffer_id_(0),
77 next_bitstream_buffer_id_(0), 128 reset_decoder_buffer_id_(ID_INVALID),
78 reset_bitstream_buffer_id_(ID_INVALID),
79 weak_factory_(this) { 129 weak_factory_(this) {
80 DCHECK(!factories_->GetTaskRunner()->BelongsToCurrentThread()); 130 DCHECK(!decoder_task_runner_->BelongsToCurrentThread());
131
132 // This object can be created on any thread, so detach now.
133 webrtc_decoding_thread_checker_.DetachFromThread();
81 } 134 }
82 135
83 RTCVideoDecoder::~RTCVideoDecoder() { 136 RTCVideoDecoder::~RTCVideoDecoder() {
84 DVLOG(2) << "~RTCVideoDecoder"; 137 DVLOG(2) << __func__;
85 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent(); 138 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
86 DestroyVDA();
87 139
88 // Delete all shared memories. 140 base::AutoLock auto_lock(lock_);
89 ClearPendingBuffers(); 141 ClearPendingBuffers_Locked();
90 } 142 }
91 143
92 // static 144 // static
93 std::unique_ptr<RTCVideoDecoder> RTCVideoDecoder::Create( 145 std::unique_ptr<RTCVideoDecoder> RTCVideoDecoder::Create(
94 webrtc::VideoCodecType type, 146 webrtc::VideoCodecType type,
95 media::GpuVideoAcceleratorFactories* factories) { 147 const CreateVideoDecoderCB& create_video_decoder_cb,
148 const scoped_refptr<base::SingleThreadTaskRunner>& decoder_task_runner) {
149 // This method posts a task to |decoder_task_runner| and blocks until it
150 // returns, so this method must not be called on the decoder thread to prevent
151 // deadlock.
152 DCHECK(!decoder_task_runner->BelongsToCurrentThread());
153
96 std::unique_ptr<RTCVideoDecoder> decoder; 154 std::unique_ptr<RTCVideoDecoder> decoder;
97 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=5717. 155 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=5717.
98 #if defined(OS_WIN) 156 #if defined(OS_WIN)
99 if (!base::CommandLine::ForCurrentProcess()->HasSwitch( 157 if (!base::CommandLine::ForCurrentProcess()->HasSwitch(
100 switches::kEnableWin7WebRtcHWH264Decoding) && 158 switches::kEnableWin7WebRtcHWH264Decoding) &&
101 type == webrtc::kVideoCodecH264 && 159 type == webrtc::kVideoCodecH264 &&
102 base::win::GetVersion() == base::win::VERSION_WIN7) { 160 base::win::GetVersion() == base::win::VERSION_WIN7) {
103 DLOG(ERROR) << "H264 HW decoding on Win7 is not supported."; 161 DLOG(ERROR) << "H264 HW decoding on Win7 is not supported.";
104 return decoder; 162 return decoder;
105 } 163 }
106 #endif // defined(OS_WIN) 164 #endif // defined(OS_WIN)
107 // Convert WebRTC codec type to media codec profile. 165
108 media::VideoCodecProfile profile; 166 // Convert WebRTC codec type to media decoder config and initialize the
109 switch (type) { 167 // decoder. Since we cannot query the remote decoder's capabilities to
110 case webrtc::kVideoCodecVP8: 168 // determine whether |type| is supported, we need to initialize the decoder
111 profile = media::VP8PROFILE_ANY; 169 // with an arbitrary size.
112 break; 170 // TODO(slan): Remove this call when it is possible to synchronously query the
113 case webrtc::kVideoCodecH264: 171 // media::VideoDecoder's capabilities (crbug.com/658349)
114 profile = media::H264PROFILE_MAIN; 172 const media::VideoDecoderConfig config = GetVideoDecoderConfigForCodec(
115 break; 173 type, gfx::Size(kDefaultVideoWidth, kDefaultVideoHeight));
116 default: 174 if (!config.IsValidConfig()) {
117 DVLOG(2) << "Video codec not supported:" << type; 175 LOG(ERROR) << "Video codec not supported:" << type;
118 return decoder; 176 return decoder;
119 } 177 }
120 178
121 base::WaitableEvent waiter(base::WaitableEvent::ResetPolicy::MANUAL, 179 decoder.reset(
122 base::WaitableEvent::InitialState::NOT_SIGNALED); 180 new RTCVideoDecoder(type, create_video_decoder_cb, decoder_task_runner));
123 decoder.reset(new RTCVideoDecoder(type, factories)); 181
124 decoder->factories_->GetTaskRunner()->PostTask( 182 // Block this thread until |decoder_| has been initialized.
125 FROM_HERE, 183 base::WaitableEvent decoder_initialized(
126 base::Bind(&RTCVideoDecoder::CreateVDA, 184 base::WaitableEvent::ResetPolicy::MANUAL,
127 base::Unretained(decoder.get()), 185 base::WaitableEvent::InitialState::NOT_SIGNALED);
128 profile, 186 decoder_task_runner->PostTask(FROM_HERE,
129 &waiter)); 187 base::Bind(&RTCVideoDecoder::InitializeDecoder,
130 waiter.Wait(); 188 base::Unretained(decoder.get()),
131 // |decoder->vda_| is nullptr if the codec is not supported. 189 &decoder_initialized, config));
132 if (decoder->vda_) 190 decoder_initialized.Wait();
133 decoder->state_ = INITIALIZED; 191 DCHECK(decoder->state_ == UNINITIALIZED || decoder->state_ == INITIALIZED);
134 else 192 if (decoder->state_ == UNINITIALIZED) {
135 factories->GetTaskRunner()->DeleteSoon(FROM_HERE, decoder.release()); 193 LOG(ERROR) << "Decoder did not initialize properly!";
194 decoder_task_runner->DeleteSoon(FROM_HERE, decoder.release());
195 }
196
136 return decoder; 197 return decoder;
137 } 198 }
138 199
139 // static 200 // static
140 void RTCVideoDecoder::Destroy(webrtc::VideoDecoder* decoder, 201 void RTCVideoDecoder::Destroy(
141 media::GpuVideoAcceleratorFactories* factories) { 202 webrtc::VideoDecoder* decoder,
142 factories->GetTaskRunner()->DeleteSoon(FROM_HERE, decoder); 203 const scoped_refptr<base::SingleThreadTaskRunner>& decoder_task_runner) {
204 CHECK(decoder_task_runner->DeleteSoon(FROM_HERE, decoder));
143 } 205 }
144 206
145 int32_t RTCVideoDecoder::InitDecode(const webrtc::VideoCodec* codecSettings, 207 int32_t RTCVideoDecoder::InitDecode(const webrtc::VideoCodec* codecSettings,
146 int32_t /*numberOfCores*/) { 208 int32_t /*numberOfCores*/) {
147 DVLOG(2) << "InitDecode"; 209 DVLOG(2) << __func__;
210 DCHECK(!decoder_task_runner_->BelongsToCurrentThread());
211 DCHECK(webrtc_decoding_thread_checker_.CalledOnValidThread());
212
148 DCHECK_EQ(video_codec_type_, codecSettings->codecType); 213 DCHECK_EQ(video_codec_type_, codecSettings->codecType);
149 if (codecSettings->codecType == webrtc::kVideoCodecVP8 && 214 if (codecSettings->codecType == webrtc::kVideoCodecVP8 &&
150 codecSettings->codecSpecific.VP8.feedbackModeOn) { 215 codecSettings->codecSpecific.VP8.feedbackModeOn) {
151 LOG(ERROR) << "Feedback mode not supported"; 216 LOG(ERROR) << "Feedback mode not supported";
152 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_ERROR); 217 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_ERROR);
153 } 218 }
154 219
220 {
221 base::AutoLock auto_lock(lock_);
222 if (state_ == DECODE_ERROR) {
223 LOG(ERROR) << "Decoder is in a bad state: " << state_;
224 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_ERROR);
225 } else if (state_ == RESETTING) {
226 LOG(ERROR) << "A Reset call is pending. OK.";
227 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_OK);
228 }
229 }
230
231 // Re-initialize the decoder with the proper frame size, even if the decoder
232 // has already been initialized.
233 const media::VideoDecoderConfig config = GetVideoDecoderConfigForCodec(
234 codecSettings->codecType,
235 gfx::Size(codecSettings->width, codecSettings->height));
236 DCHECK(config.IsValidConfig());
237 base::WaitableEvent decoder_initialized(
238 base::WaitableEvent::ResetPolicy::MANUAL,
239 base::WaitableEvent::InitialState::NOT_SIGNALED);
240 decoder_task_runner_->PostTask(
241 FROM_HERE,
242 base::Bind(&RTCVideoDecoder::InitializeDecoder, base::Unretained(this),
243 &decoder_initialized, config));
244 decoder_initialized.Wait();
245
155 base::AutoLock auto_lock(lock_); 246 base::AutoLock auto_lock(lock_);
156 if (state_ == UNINITIALIZED || state_ == DECODE_ERROR) { 247 if (state_ == UNINITIALIZED) {
157 LOG(ERROR) << "VDA is not initialized. state=" << state_; 248 LOG(ERROR) << "Decoder failed to reinitialize correctly!";
158 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_UNINITIALIZED); 249 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_UNINITIALIZED);
159 } 250 }
160
161 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_OK); 251 return RecordInitDecodeUMA(WEBRTC_VIDEO_CODEC_OK);
162 } 252 }
163 253
164 int32_t RTCVideoDecoder::Decode( 254 int32_t RTCVideoDecoder::Decode(
165 const webrtc::EncodedImage& inputImage, 255 const webrtc::EncodedImage& inputImage,
166 bool missingFrames, 256 bool missingFrames,
167 const webrtc::RTPFragmentationHeader* /*fragmentation*/, 257 const webrtc::RTPFragmentationHeader* /*fragmentation*/,
168 const webrtc::CodecSpecificInfo* /*codecSpecificInfo*/, 258 const webrtc::CodecSpecificInfo* /*codecSpecificInfo*/,
169 int64_t /*renderTimeMs*/) { 259 int64_t /*renderTimeMs*/) {
170 DVLOG(3) << "Decode"; 260 DVLOG(2) << __func__;
261 DCHECK(!decoder_task_runner_->BelongsToCurrentThread());
262 DCHECK(webrtc_decoding_thread_checker_.CalledOnValidThread());
171 263
172 base::AutoLock auto_lock(lock_); 264 base::AutoLock auto_lock(lock_);
173 265
174 if (state_ == UNINITIALIZED || !decode_complete_callback_) { 266 if (!decode_complete_callback_) {
175 LOG(ERROR) << "The decoder has not initialized."; 267 LOG(ERROR) << "The decoder has not initialized.";
176 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 268 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
177 } 269 }
178 270
179 if (state_ == DECODE_ERROR) { 271 if (state_ == DECODE_ERROR) {
180 LOG(ERROR) << "Decoding error occurred."; 272 LOG(ERROR) << "Decoding error occurred.";
181 // Try reseting the session up to |kNumVDAErrorsHandled| times. 273 // Try reseting the session up to |kNumDecoderErrorsHandled| times.
182 // Check if SW H264 implementation is available before falling back. 274 // Check if SW H264 implementation is available before falling back.
183 if (vda_error_counter_ > kNumVDAErrorsBeforeSWFallback && 275 if (decoder_error_counter_ > kNumDecoderErrorsBeforeSWFallback &&
184 (video_codec_type_ != webrtc::kVideoCodecH264 || 276 (video_codec_type_ != webrtc::kVideoCodecH264 ||
185 webrtc::H264Decoder::IsSupported())) { 277 webrtc::H264Decoder::IsSupported())) {
186 DLOG(ERROR) << vda_error_counter_ 278 DLOG(ERROR)
187 << " errors reported by VDA, falling back to software decode"; 279 << decoder_error_counter_
280 << " errors reported by decoder, falling back to software decode";
188 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; 281 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
189 } 282 }
190 base::AutoUnlock auto_unlock(lock_); 283 base::AutoUnlock auto_unlock(lock_);
191 Release(); 284 Release();
192 return WEBRTC_VIDEO_CODEC_ERROR; 285 return WEBRTC_VIDEO_CODEC_ERROR;
193 } 286 }
194 287
195 if (missingFrames || !inputImage._completeFrame) { 288 if (missingFrames || !inputImage._completeFrame) {
196 DLOG(ERROR) << "Missing or incomplete frames."; 289 LOG(ERROR) << "Missing or incomplete frames.";
197 // Unlike the SW decoder in libvpx, hw decoder cannot handle broken frames. 290 // Unlike the SW decoder in libvpx, hw decoder cannot handle broken frames.
198 // Return an error to request a key frame. 291 // Return an error to request a key frame.
199 return WEBRTC_VIDEO_CODEC_ERROR; 292 return WEBRTC_VIDEO_CODEC_ERROR;
200 } 293 }
201 294
202 // Most platforms' VDA implementations support mid-stream resolution change 295 // Most platforms' VDA implementations support mid-stream resolution change
203 // internally. Platforms whose VDAs fail to support mid-stream resolution 296 // internally. Platforms whose VDAs fail to support mid-stream resolution
204 // change gracefully need to have their clients cover for them, and we do that 297 // change gracefully need to have their clients cover for them, and we do that
205 // here. 298 // here.
206 #ifdef ANDROID 299 #ifdef ANDROID
207 const bool kVDACanHandleMidstreamResize = false; 300 const bool kDecoderCanHandleMidstreamResize = false;
208 #else 301 #else
209 const bool kVDACanHandleMidstreamResize = true; 302 const bool kDecoderCanHandleMidstreamResize = true;
210 #endif 303 #endif
211 304
212 bool need_to_reset_for_midstream_resize = false; 305 bool need_to_reset_for_midstream_resize = false;
213 if (inputImage._frameType == webrtc::kVideoFrameKey) { 306 if (inputImage._frameType == webrtc::kVideoFrameKey) {
214 const gfx::Size new_frame_size(inputImage._encodedWidth, 307 const gfx::Size new_frame_size(inputImage._encodedWidth,
215 inputImage._encodedHeight); 308 inputImage._encodedHeight);
216 DVLOG(2) << "Got key frame. size=" << new_frame_size.ToString(); 309 DVLOG(1) << "Got key frame. size=" << new_frame_size.ToString();
217 310
218 if (new_frame_size.width() > max_resolution_.width() || 311 // TODO(slan): Query beckend for capabilities and fallback to software
219 new_frame_size.width() < min_resolution_.width() || 312 // immediately if the resolution is out of bounds. Currently, we'll have to
220 new_frame_size.height() > max_resolution_.height() || 313 // wait until the decoder calls back with a DECODE_ERROR to tell us this.
221 new_frame_size.height() < min_resolution_.height()) {
222 DVLOG(1) << "Resolution unsupported, falling back to software decode";
223 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
224 }
225 314
226 gfx::Size prev_frame_size = frame_size_; 315 gfx::Size prev_frame_size = frame_size_;
227 frame_size_ = new_frame_size; 316 frame_size_ = new_frame_size;
228 if (!kVDACanHandleMidstreamResize && !prev_frame_size.IsEmpty() && 317 if (!kDecoderCanHandleMidstreamResize && !prev_frame_size.IsEmpty() &&
229 prev_frame_size != frame_size_) { 318 prev_frame_size != frame_size_) {
230 need_to_reset_for_midstream_resize = true; 319 need_to_reset_for_midstream_resize = true;
231 } 320 }
232 } else if (IsFirstBufferAfterReset(next_bitstream_buffer_id_, 321 } else if (IsFirstBufferAfterReset(next_decoder_buffer_id_,
233 reset_bitstream_buffer_id_)) { 322 reset_decoder_buffer_id_)) {
234 // TODO(wuchengli): VDA should handle it. Remove this when 323 // The first frame after a reset should be a key frame. If we are in an
235 // http://crosbug.com/p/21913 is fixed. 324 // error condition, increase the counter.
236 325 LOG(ERROR) << "The first frame should be a key frame. Drop this.";
237 // If we're are in an error condition, increase the counter. 326 decoder_error_counter_ += decoder_error_counter_ ? 1 : 0;
238 vda_error_counter_ += vda_error_counter_ ? 1 : 0;
239
240 DVLOG(1) << "The first frame should be a key frame. Drop this.";
241 return WEBRTC_VIDEO_CODEC_ERROR; 327 return WEBRTC_VIDEO_CODEC_ERROR;
242 } 328 }
243 329
244 // Create buffer metadata. 330 const int32_t decoder_buffer_id = next_decoder_buffer_id_;
245 BufferData buffer_data(next_bitstream_buffer_id_, 331 auto decoder_buffer = CreateDecoderBuffer(inputImage);
246 inputImage._timeStamp, 332
247 inputImage._length,
248 gfx::Rect(frame_size_));
249 // Mask against 30 bits, to avoid (undefined) wraparound on signed integer. 333 // Mask against 30 bits, to avoid (undefined) wraparound on signed integer.
250 next_bitstream_buffer_id_ = (next_bitstream_buffer_id_ + 1) & ID_LAST; 334 next_decoder_buffer_id_ = (next_decoder_buffer_id_ + 1) & ID_LAST;
251 335
252 // If a shared memory segment is available, there are no pending buffers, and 336 // Try to enqueue the image to be decoded. These frames will be consumed on
253 // this isn't a mid-stream resolution change, then send the buffer for decode 337 // the decoder thread. It would be ideal to post this to the decoder thread to
254 // immediately. Otherwise, save the buffer in the queue for later decode. 338 // avoid the lock, but we need to return an error from this function if
255 std::unique_ptr<base::SharedMemory> shm_buffer; 339 // |pending_buffers_| is full. So use a lock instead.
256 if (!need_to_reset_for_midstream_resize && pending_buffers_.empty()) 340 if (!SaveToPendingBuffers_Locked(decoder_buffer_id,
257 shm_buffer = GetSHM_Locked(inputImage._length); 341 std::move(decoder_buffer))) {
258 if (!shm_buffer) { 342 // We have exceeded the pending buffers count, we are severely behind.
259 if (!SaveToPendingBuffers_Locked(inputImage, buffer_data)) { 343 // Since we are returning ERROR, WebRTC will not be interested in the
260 // We have exceeded the pending buffers count, we are severely behind. 344 // remaining buffers, and will provide us with a new keyframe instead.
261 // Since we are returning ERROR, WebRTC will not be interested in the 345 // Better to drop any pending buffers and start afresh to catch up faster.
262 // remaining buffers, and will provide us with a new keyframe instead. 346 LOG(ERROR) << "Exceeded maximum pending buffer count, dropping";
263 // Better to drop any pending buffers and start afresh to catch up faster. 347 ClearPendingBuffers_Locked();
264 DVLOG(1) << "Exceeded maximum pending buffer count, dropping"; 348 return WEBRTC_VIDEO_CODEC_ERROR;
265 ClearPendingBuffers(); 349 }
266 return WEBRTC_VIDEO_CODEC_ERROR; 350
267 } 351 if (need_to_reset_for_midstream_resize) {
268 352 base::AutoUnlock auto_unlock(lock_);
269 if (need_to_reset_for_midstream_resize) { 353 Release();
270 base::AutoUnlock auto_unlock(lock_); 354 }
271 Release(); 355
272 } 356 // If the decoder has been initialized, request a decode.
273 return WEBRTC_VIDEO_CODEC_OK; 357 if (state_ == INITIALIZED) {
274 } 358 decoder_task_runner_->PostTask(
275 359 FROM_HERE, base::Bind(&RTCVideoDecoder::RequestBufferDecode,
276 SaveToDecodeBuffers_Locked(inputImage, std::move(shm_buffer), buffer_data); 360 weak_factory_.GetWeakPtr()));
277 factories_->GetTaskRunner()->PostTask( 361 }
278 FROM_HERE,
279 base::Bind(&RTCVideoDecoder::RequestBufferDecode,
280 weak_factory_.GetWeakPtr()));
281 return WEBRTC_VIDEO_CODEC_OK; 362 return WEBRTC_VIDEO_CODEC_OK;
282 } 363 }
283 364
284 int32_t RTCVideoDecoder::RegisterDecodeCompleteCallback( 365 int32_t RTCVideoDecoder::RegisterDecodeCompleteCallback(
285 webrtc::DecodedImageCallback* callback) { 366 webrtc::DecodedImageCallback* callback) {
286 DVLOG(2) << "RegisterDecodeCompleteCallback"; 367 DVLOG(2) << __func__;
368 DCHECK(webrtc_decoding_thread_checker_.CalledOnValidThread());
287 DCHECK(callback); 369 DCHECK(callback);
370
288 base::AutoLock auto_lock(lock_); 371 base::AutoLock auto_lock(lock_);
289 decode_complete_callback_ = callback; 372 decode_complete_callback_ = callback;
290 return WEBRTC_VIDEO_CODEC_OK; 373 return WEBRTC_VIDEO_CODEC_OK;
291 } 374 }
292 375
293 int32_t RTCVideoDecoder::Release() { 376 int32_t RTCVideoDecoder::Release() {
294 DVLOG(2) << "Release"; 377 DVLOG(2) << __func__;
295 // Do not destroy VDA because WebRTC can call InitDecode and start decoding 378 DCHECK(!decoder_task_runner_->BelongsToCurrentThread());
296 // again. 379
297 base::AutoLock auto_lock(lock_); 380 base::AutoLock auto_lock(lock_);
298 if (state_ == UNINITIALIZED) { 381 if (state_ == UNINITIALIZED) {
299 LOG(ERROR) << "Decoder not initialized."; 382 LOG(ERROR) << "Decoder not initialized.";
300 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 383 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
301 } 384 }
302 if (next_bitstream_buffer_id_ != 0) 385
303 reset_bitstream_buffer_id_ = next_bitstream_buffer_id_ - 1; 386 if (next_decoder_buffer_id_ != 0)
387 reset_decoder_buffer_id_ = next_decoder_buffer_id_ - 1;
304 else 388 else
305 reset_bitstream_buffer_id_ = ID_LAST; 389 reset_decoder_buffer_id_ = ID_LAST;
306 // If VDA is already resetting, no need to request the reset again. 390
391 // If the decoder is already resetting, no need to request the reset again.
307 if (state_ != RESETTING) { 392 if (state_ != RESETTING) {
308 state_ = RESETTING; 393 state_ = RESETTING;
309 factories_->GetTaskRunner()->PostTask( 394
310 FROM_HERE, 395 // Reset |decoder_| on |decoder_task_runner_|. Until the callback returns,
311 base::Bind(&RTCVideoDecoder::ResetInternal, 396 // no more calls should be made on |decoder_|.
312 weak_factory_.GetWeakPtr())); 397 decoder_task_runner_->PostTask(
398 FROM_HERE, base::Bind(&media::VideoDecoder::Reset,
399 base::Unretained(decoder_.get()),
400 base::Bind(&RTCVideoDecoder::OnResetDone,
401 weak_factory_.GetWeakPtr())));
313 } 402 }
314 return WEBRTC_VIDEO_CODEC_OK; 403 return WEBRTC_VIDEO_CODEC_OK;
315 } 404 }
316 405
317 void RTCVideoDecoder::ProvidePictureBuffers(uint32_t count, 406 void RTCVideoDecoder::OnResetDone() {
318 media::VideoPixelFormat format, 407 DVLOG(2) << __func__;
319 uint32_t textures_per_buffer, 408 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
320 const gfx::Size& size,
321 uint32_t texture_target) {
322 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
323 DVLOG(3) << "ProvidePictureBuffers. texture_target=" << texture_target;
324 DCHECK_EQ(1u, textures_per_buffer);
325
326 if (!vda_)
327 return;
328
329 std::vector<uint32_t> texture_ids;
330 std::vector<gpu::Mailbox> texture_mailboxes;
331 decoder_texture_target_ = texture_target;
332
333 if (format == media::PIXEL_FORMAT_UNKNOWN)
334 format = media::PIXEL_FORMAT_ARGB;
335
336 if ((pixel_format_ != media::PIXEL_FORMAT_UNKNOWN) &&
337 (format != pixel_format_)) {
338 NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
339 return;
340 }
341
342 pixel_format_ = format;
343 if (!factories_->CreateTextures(count,
344 size,
345 &texture_ids,
346 &texture_mailboxes,
347 decoder_texture_target_)) {
348 NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
349 return;
350 }
351 DCHECK_EQ(count, texture_ids.size());
352 DCHECK_EQ(count, texture_mailboxes.size());
353
354 std::vector<media::PictureBuffer> picture_buffers;
355 for (size_t i = 0; i < texture_ids.size(); ++i) {
356 media::PictureBuffer::TextureIds ids;
357 ids.push_back(texture_ids[i]);
358 std::vector<gpu::Mailbox> mailboxes;
359 mailboxes.push_back(texture_mailboxes[i]);
360
361 picture_buffers.push_back(
362 media::PictureBuffer(next_picture_buffer_id_++, size, ids, mailboxes));
363 bool inserted = assigned_picture_buffers_.insert(std::make_pair(
364 picture_buffers.back().id(), picture_buffers.back())).second;
365 DCHECK(inserted);
366 }
367 vda_->AssignPictureBuffers(picture_buffers);
368 }
369
370 void RTCVideoDecoder::DismissPictureBuffer(int32_t id) {
371 DVLOG(3) << "DismissPictureBuffer. id=" << id;
372 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
373
374 std::map<int32_t, media::PictureBuffer>::iterator it =
375 assigned_picture_buffers_.find(id);
376 if (it == assigned_picture_buffers_.end()) {
377 NOTREACHED() << "Missing picture buffer: " << id;
378 return;
379 }
380
381 media::PictureBuffer buffer_to_dismiss = it->second;
382 assigned_picture_buffers_.erase(it);
383
384 if (!picture_buffers_at_display_.count(id)) {
385 // We can delete the texture immediately as it's not being displayed.
386 factories_->DeleteTexture(buffer_to_dismiss.texture_ids()[0]);
387 return;
388 }
389 // Not destroying a texture in display in |picture_buffers_at_display_|.
390 // Postpone deletion until after it's returned to us.
391 }
392
393 void RTCVideoDecoder::PictureReady(const media::Picture& picture) {
394 DVLOG(3) << "PictureReady";
395 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
396
397 std::map<int32_t, media::PictureBuffer>::iterator it =
398 assigned_picture_buffers_.find(picture.picture_buffer_id());
399 if (it == assigned_picture_buffers_.end()) {
400 NOTREACHED() << "Missing picture buffer: " << picture.picture_buffer_id();
401 NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
402 return;
403 }
404
405 uint32_t timestamp = 0;
406 gfx::Rect visible_rect;
407 GetBufferData(picture.bitstream_buffer_id(), &timestamp, &visible_rect);
408 if (!picture.visible_rect().IsEmpty())
409 visible_rect = picture.visible_rect();
410
411 const media::PictureBuffer& pb = it->second;
412 if (visible_rect.IsEmpty() || !gfx::Rect(pb.size()).Contains(visible_rect)) {
413 LOG(ERROR) << "Invalid picture size: " << visible_rect.ToString()
414 << " should fit in " << pb.size().ToString();
415 NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
416 return;
417 }
418
419 scoped_refptr<media::VideoFrame> frame =
420 CreateVideoFrame(picture, pb, timestamp, visible_rect, pixel_format_);
421 if (!frame) {
422 NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
423 return;
424 }
425 bool inserted = picture_buffers_at_display_
426 .insert(std::make_pair(picture.picture_buffer_id(),
427 pb.texture_ids()[0]))
428 .second;
429 DCHECK(inserted);
430
431 // Create a WebRTC video frame.
432 webrtc::VideoFrame decoded_image(
433 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>(frame), timestamp, 0,
434 webrtc::kVideoRotation_0);
435
436 // Invoke decode callback. WebRTC expects no callback after Release.
437 {
438 base::AutoLock auto_lock(lock_);
439 DCHECK(decode_complete_callback_);
440 if (IsBufferAfterReset(picture.bitstream_buffer_id(),
441 reset_bitstream_buffer_id_)) {
442 decode_complete_callback_->Decoded(decoded_image);
443 }
444 // Reset error counter as we successfully decoded a frame.
445 vda_error_counter_ = 0;
446 }
447 }
448
449 scoped_refptr<media::VideoFrame> RTCVideoDecoder::CreateVideoFrame(
450 const media::Picture& picture,
451 const media::PictureBuffer& pb,
452 uint32_t timestamp,
453 const gfx::Rect& visible_rect,
454 media::VideoPixelFormat pixel_format) {
455 DCHECK(decoder_texture_target_);
456 // Convert timestamp from 90KHz to ms.
457 base::TimeDelta timestamp_ms = base::TimeDelta::FromInternalValue(
458 base::checked_cast<uint64_t>(timestamp) * 1000 / 90);
459 // TODO(mcasas): The incoming data may actually be in a YUV format, but may be
460 // labelled as ARGB. This may or may not be reported by VDA, depending on
461 // whether it provides an implementation of VDA::GetOutputFormat().
462 // This prevents the compositor from messing with it, since the underlying
463 // platform can handle the former format natively. Make sure the
464 // correct format is used and everyone down the line understands it.
465 gpu::MailboxHolder holders[media::VideoFrame::kMaxPlanes] = {
466 gpu::MailboxHolder(pb.texture_mailbox(0), gpu::SyncToken(),
467 decoder_texture_target_)};
468 scoped_refptr<media::VideoFrame> frame =
469 media::VideoFrame::WrapNativeTextures(
470 pixel_format, holders,
471 media::BindToCurrentLoop(base::Bind(
472 &RTCVideoDecoder::ReleaseMailbox, weak_factory_.GetWeakPtr(),
473 factories_, picture.picture_buffer_id(), pb.texture_ids()[0])),
474 pb.size(), visible_rect, visible_rect.size(), timestamp_ms);
475 if (frame && picture.allow_overlay()) {
476 frame->metadata()->SetBoolean(media::VideoFrameMetadata::ALLOW_OVERLAY,
477 true);
478 }
479 return frame;
480 }
481
482 void RTCVideoDecoder::NotifyEndOfBitstreamBuffer(int32_t id) {
483 DVLOG(3) << "NotifyEndOfBitstreamBuffer. id=" << id;
484 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
485
486 auto it = bitstream_buffers_in_decoder_.find(id);
487 if (it == bitstream_buffers_in_decoder_.end()) {
488 NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
489 NOTREACHED() << "Missing bitstream buffer: " << id;
490 return;
491 }
492
493 {
494 base::AutoLock auto_lock(lock_);
495 PutSHM_Locked(std::move(it->second));
496 }
497 bitstream_buffers_in_decoder_.erase(it);
498
499 RequestBufferDecode();
500 }
501
502 void RTCVideoDecoder::NotifyFlushDone() {
503 DVLOG(3) << "NotifyFlushDone";
504 NOTREACHED() << "Unexpected flush done notification.";
505 }
506
507 void RTCVideoDecoder::NotifyResetDone() {
508 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
509 DVLOG(3) << "NotifyResetDone";
510
511 if (!vda_)
512 return;
513
514 input_buffer_data_.clear();
515 { 409 {
516 base::AutoLock auto_lock(lock_); 410 base::AutoLock auto_lock(lock_);
517 state_ = INITIALIZED; 411 state_ = INITIALIZED;
518 } 412 }
519 // Send the pending buffers for decoding. 413
414 // Send any pending buffers for decoding.
520 RequestBufferDecode(); 415 RequestBufferDecode();
521 } 416 }
522 417
523 void RTCVideoDecoder::NotifyError(media::VideoDecodeAccelerator::Error error) { 418 void RTCVideoDecoder::OnBufferDecoded(int32_t decoder_buffer_id,
524 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent(); 419 media::DecodeStatus status) {
525 if (!vda_) 420 DVLOG(2) << __func__;
421 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
422
423 // This buffer is no longer in flight. Remove it from the in-flight buffers.
424 bool erased = buffers_in_decoder_.erase(decoder_buffer_id);
425 DCHECK(erased);
426
427 // This is only called when |decoder_| is torn down while buffers are in
428 // flight. This probably indicates that something has gone wrong, so let's
429 // do NOTREACHED() now and figure out how to handle this later.
430 if (status == media::DecodeStatus::ABORTED) {
431 NOTREACHED();
432 } else if (status == media::DecodeStatus::OK) {
433 decoder_task_runner_->PostTask(
434 FROM_HERE, base::Bind(&RTCVideoDecoder::RequestBufferDecode,
435 weak_factory_.GetWeakPtr()));
526 return; 436 return;
527 437 }
528 LOG(ERROR) << "VDA Error:" << error; 438
529 UMA_HISTOGRAM_ENUMERATION("Media.RTCVideoDecoderError", error, 439 // If we hit here, |decoder_| is bubbling up an error from the remote decoder.
530 media::VideoDecodeAccelerator::ERROR_MAX + 1); 440 DCHECK(status == media::DecodeStatus::DECODE_ERROR);
531 DestroyVDA(); 441 LOG(ERROR) << "Decoder encountered an error.";
532 442
533 base::AutoLock auto_lock(lock_); 443 // TODO(slan): Do we need an UMA stat?
534 state_ = DECODE_ERROR; 444
535 ++vda_error_counter_; 445 // Re-enqueue patches in-flight so we can attempt to decode them again. Insert
446 // them back into the queue in their original order.
447 std::map<int32_t, scoped_refptr<media::DecoderBuffer>> sorted(
448 buffers_in_decoder_.begin(), buffers_in_decoder_.end());
449 {
450 base::AutoLock auto_lock(lock_);
451 for (auto rit = sorted.rbegin(); rit != sorted.rend(); ++rit)
452 pending_buffers_.push_front(std::make_pair(rit->first, rit->second));
453
454 state_ = DECODE_ERROR;
455 ++decoder_error_counter_;
456 }
536 } 457 }
537 458
538 void RTCVideoDecoder::RequestBufferDecode() { 459 void RTCVideoDecoder::RequestBufferDecode() {
539 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent(); 460 DVLOG(2) << __func__;
540 if (!vda_) 461 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
541 return; 462
542 463 // If less than the maximum possible number of decode requests are in flight,
543 MovePendingBuffersToDecodeBuffers(); 464 // we might be able to push more into the decoder.
544
545 while (CanMoreDecodeWorkBeDone()) { 465 while (CanMoreDecodeWorkBeDone()) {
546 // Get a buffer and data from the queue. 466 int32_t decoder_buffer_id = ID_INVALID;
547 std::unique_ptr<base::SharedMemory> shm_buffer; 467 scoped_refptr<media::DecoderBuffer> decoder_buffer;
548 BufferData buffer_data;
549 { 468 {
550 base::AutoLock auto_lock(lock_); 469 base::AutoLock auto_lock(lock_);
551 // Do not request decode if VDA is resetting. 470
552 if (decode_buffers_.empty() || state_ == RESETTING) 471 // Do not request decode if decoder_ is resetting or if there are no
472 // buffers to decode.
473 if (state_ == RESETTING || pending_buffers_.empty())
553 return; 474 return;
554 shm_buffer = std::move(decode_buffers_.front().first); 475
555 buffer_data = decode_buffers_.front().second; 476 // Remove the buffer from the queue.
556 decode_buffers_.pop_front(); 477 decoder_buffer_id = pending_buffers_.front().first;
557 // Drop the buffers before Release is called. 478 decoder_buffer = std::move(pending_buffers_.front().second);
558 if (!IsBufferAfterReset(buffer_data.bitstream_buffer_id, 479 pending_buffers_.pop_front();
559 reset_bitstream_buffer_id_)) {
560 PutSHM_Locked(std::move(shm_buffer));
561 continue;
562 }
563 } 480 }
564 481
565 // Create a BitstreamBuffer and send to VDA to decode. 482 // If the buffer is from before the last call to Release(), drop it on
566 media::BitstreamBuffer bitstream_buffer( 483 // the floor and keep going.
567 buffer_data.bitstream_buffer_id, shm_buffer->handle(), buffer_data.size, 484 if (!IsBufferAfterReset(decoder_buffer_id, reset_decoder_buffer_id_)) {
568 0, base::TimeDelta::FromInternalValue(buffer_data.timestamp)); 485 continue;
569 const bool inserted = bitstream_buffers_in_decoder_ 486 }
570 .insert(std::make_pair(bitstream_buffer.id(), 487
571 std::move(shm_buffer))) 488 // Save a reference to |decoder_buffer| in |buffers_in_decoder_|.
572 .second; 489 DCHECK(!base::ContainsKey(buffers_in_decoder_, decoder_buffer_id));
573 DCHECK(inserted) << "bitstream_buffer_id " << bitstream_buffer.id() 490 buffers_in_decoder_[decoder_buffer_id] = decoder_buffer;
574 << " existed already in bitstream_buffers_in_decoder_"; 491
575 RecordBufferData(buffer_data); 492 // Push the buffer to the decoder.
576 vda_->Decode(bitstream_buffer); 493 decoder_->Decode(std::move(decoder_buffer),
494 base::Bind(&RTCVideoDecoder::OnBufferDecoded,
495 weak_factory_.GetWeakPtr(), decoder_buffer_id));
577 } 496 }
578 } 497 }
579 498
580 bool RTCVideoDecoder::CanMoreDecodeWorkBeDone() { 499 bool RTCVideoDecoder::CanMoreDecodeWorkBeDone() {
581 return bitstream_buffers_in_decoder_.size() < kMaxInFlightDecodes; 500 DVLOG(2) << __func__;
582 } 501 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
583 502 return (static_cast<int>(buffers_in_decoder_.size()) <
584 bool RTCVideoDecoder::IsBufferAfterReset(int32_t id_buffer, int32_t id_reset) { 503 decoder_->GetMaxDecodeRequests());
585 if (id_reset == ID_INVALID) 504 }
586 return true; 505
587 int32_t diff = id_buffer - id_reset; 506 int RTCVideoDecoder::GetDecoderErrorCounterForTesting() {
588 if (diff <= 0) 507 base::AutoLock auto_lock(lock_);
589 diff += ID_LAST + 1; 508 return decoder_error_counter_;
590 return diff < ID_HALF;
591 }
592
593 bool RTCVideoDecoder::IsFirstBufferAfterReset(int32_t id_buffer,
594 int32_t id_reset) {
595 if (id_reset == ID_INVALID)
596 return id_buffer == 0;
597 return id_buffer == ((id_reset + 1) & ID_LAST);
598 }
599
600 void RTCVideoDecoder::SaveToDecodeBuffers_Locked(
601 const webrtc::EncodedImage& input_image,
602 std::unique_ptr<base::SharedMemory> shm_buffer,
603 const BufferData& buffer_data) {
604 memcpy(shm_buffer->memory(), input_image._buffer, input_image._length);
605
606 // Store the buffer and the metadata to the queue.
607 decode_buffers_.emplace_back(std::move(shm_buffer), buffer_data);
608 } 509 }
609 510
610 bool RTCVideoDecoder::SaveToPendingBuffers_Locked( 511 bool RTCVideoDecoder::SaveToPendingBuffers_Locked(
611 const webrtc::EncodedImage& input_image, 512 int32_t decoder_buffer_id,
612 const BufferData& buffer_data) { 513 scoped_refptr<media::DecoderBuffer> decoder_buffer) {
613 DVLOG(2) << "SaveToPendingBuffers_Locked" 514 DVLOG(2) << __func__ << " pending_buffers size=" << pending_buffers_.size();
614 << ". pending_buffers size=" << pending_buffers_.size() 515 lock_.AssertAcquired();
615 << ". decode_buffers_ size=" << decode_buffers_.size() 516
616 << ". available_shm size=" << available_shm_segments_.size(); 517 // Queued too many buffers. Something has gone wrong.
617 // Queued too many buffers. Something goes wrong.
618 if (pending_buffers_.size() >= kMaxNumOfPendingBuffers) { 518 if (pending_buffers_.size() >= kMaxNumOfPendingBuffers) {
619 LOG(WARNING) << "Too many pending buffers!"; 519 LOG(WARNING) << "Too many pending buffers!";
620 return false; 520 return false;
621 } 521 }
622 522
623 // Clone the input image and save it to the queue. 523 // Enqueue the buffer, so that it may be consumed by the decoder.
624 uint8_t* buffer = new uint8_t[input_image._length]; 524 pending_buffers_.push_back(
625 // TODO(wuchengli): avoid memcpy. Extend webrtc::VideoDecoder::Decode() 525 std::make_pair(decoder_buffer_id, std::move(decoder_buffer)));
626 // interface to take a non-const ptr to the frame and add a method to the
627 // frame that will swap buffers with another.
628 memcpy(buffer, input_image._buffer, input_image._length);
629 webrtc::EncodedImage encoded_image(
630 buffer, input_image._length, input_image._length);
631 std::pair<webrtc::EncodedImage, BufferData> buffer_pair =
632 std::make_pair(encoded_image, buffer_data);
633
634 pending_buffers_.push_back(buffer_pair);
635 return true; 526 return true;
636 } 527 }
637 528
638 void RTCVideoDecoder::MovePendingBuffersToDecodeBuffers() { 529 void RTCVideoDecoder::InitializeDecoder(
639 base::AutoLock auto_lock(lock_); 530 base::WaitableEvent* decoder_initialized,
640 while (pending_buffers_.size() > 0) { 531 const media::VideoDecoderConfig& config) {
641 // Get a pending buffer from the queue. 532 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
642 const webrtc::EncodedImage& input_image = pending_buffers_.front().first; 533
643 const BufferData& buffer_data = pending_buffers_.front().second; 534 if (!decoder_)
644 535 decoder_ = create_video_decoder_cb_.Run();
645 // Drop the frame if it comes before Release. 536
646 if (!IsBufferAfterReset(buffer_data.bitstream_buffer_id, 537 decoder_->Initialize(
647 reset_bitstream_buffer_id_)) { 538 config, true /* low_delay */, nullptr /* cdm_context */,
648 delete[] input_image._buffer; 539 base::Bind(&RTCVideoDecoder::OnDecoderInitialized,
649 pending_buffers_.pop_front(); 540 weak_factory_.GetWeakPtr(), decoder_initialized),
650 continue; 541 base::Bind(&RTCVideoDecoder::OnFrameReady, weak_factory_.GetWeakPtr()));
651 } 542 }
652 // Get shared memory and save it to decode buffers. 543
653 std::unique_ptr<base::SharedMemory> shm_buffer = 544 void RTCVideoDecoder::OnDecoderInitialized(
654 GetSHM_Locked(input_image._length); 545 base::WaitableEvent* decoder_initialized,
655 if (!shm_buffer) 546 bool success) {
656 return; 547 DVLOG(2) << __func__;
657 SaveToDecodeBuffers_Locked(input_image, std::move(shm_buffer), buffer_data); 548 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
658 delete[] input_image._buffer; 549
659 pending_buffers_.pop_front(); 550 {
660 } 551 base::AutoLock lock(lock_);
661 } 552 state_ = success ? INITIALIZED : UNINITIALIZED;
662 553 LOG_IF(ERROR, !success) << "Decoder failed to initialize!";
663 void RTCVideoDecoder::ResetInternal() { 554 }
664 DVLOG(2) << __func__; 555
665 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent(); 556 // Unblock the waiting thread.
666 557 decoder_initialized->Signal();
667 if (vda_) { 558 }
668 vda_->Reset(); 559
669 } else { 560 void RTCVideoDecoder::OnFrameReady(
670 CreateVDA(vda_codec_profile_, nullptr); 561 const scoped_refptr<media::VideoFrame>& frame) {
671 if (vda_) { 562 DVLOG(2) << __func__;
672 base::AutoLock auto_lock(lock_); 563 DCHECK(decoder_task_runner_->BelongsToCurrentThread());
673 state_ = INITIALIZED; 564
674 } 565 // Create a WebRTC video frame.
675 } 566 webrtc::VideoFrame decoded_image(
676 } 567 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>(frame),
677 568 frame->timestamp().ToInternalValue(), 0, webrtc::kVideoRotation_0);
678 // static 569
679 void RTCVideoDecoder::ReleaseMailbox( 570 // Lock and pass the frame up to the WebRTC client class.
680 base::WeakPtr<RTCVideoDecoder> decoder, 571 {
681 media::GpuVideoAcceleratorFactories* factories,
682 int64_t picture_buffer_id,
683 uint32_t texture_id,
684 const gpu::SyncToken& release_sync_token) {
685 DCHECK(factories->GetTaskRunner()->BelongsToCurrentThread());
686 factories->WaitSyncToken(release_sync_token);
687
688 if (decoder) {
689 decoder->ReusePictureBuffer(picture_buffer_id);
690 return;
691 }
692 // It's the last chance to delete the texture after display,
693 // because RTCVideoDecoder was destructed.
694 factories->DeleteTexture(texture_id);
695 }
696
697 void RTCVideoDecoder::ReusePictureBuffer(int64_t picture_buffer_id) {
698 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
699 DVLOG(3) << "ReusePictureBuffer. id=" << picture_buffer_id;
700
701 DCHECK(!picture_buffers_at_display_.empty());
702 PictureBufferTextureMap::iterator display_iterator =
703 picture_buffers_at_display_.find(picture_buffer_id);
704 uint32_t texture_id = display_iterator->second;
705 DCHECK(display_iterator != picture_buffers_at_display_.end());
706 picture_buffers_at_display_.erase(display_iterator);
707
708 if (!assigned_picture_buffers_.count(picture_buffer_id)) {
709 // This picture was dismissed while in display, so we postponed deletion.
710 factories_->DeleteTexture(texture_id);
711 return;
712 }
713
714 // DestroyVDA() might already have been called.
715 if (vda_)
716 vda_->ReusePictureBuffer(picture_buffer_id);
717 }
718
719 bool RTCVideoDecoder::IsProfileSupported(media::VideoCodecProfile profile) {
720 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
721 media::VideoDecodeAccelerator::Capabilities capabilities =
722 factories_->GetVideoDecodeAcceleratorCapabilities();
723
724 for (const auto& supported_profile : capabilities.supported_profiles) {
725 if (profile == supported_profile.profile) {
726 min_resolution_ = supported_profile.min_resolution;
727 max_resolution_ = supported_profile.max_resolution;
728 return true;
729 }
730 }
731
732 return false;
733 }
734
735 void RTCVideoDecoder::CreateVDA(media::VideoCodecProfile profile,
736 base::WaitableEvent* waiter) {
737 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
738
739 if (!IsProfileSupported(profile)) {
740 DVLOG(1) << "Unsupported profile " << profile;
741 } else {
742 vda_ = factories_->CreateVideoDecodeAccelerator();
743
744 media::VideoDecodeAccelerator::Config config(profile);
745 if (vda_ && !vda_->Initialize(config, this))
746 vda_.release()->Destroy();
747 vda_codec_profile_ = profile;
748 }
749
750 if (waiter)
751 waiter->Signal();
752 }
753
754 void RTCVideoDecoder::DestroyTextures() {
755 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
756
757 // Not destroying PictureBuffers in |picture_buffers_at_display_| yet, since
758 // their textures may still be in use by the user of this RTCVideoDecoder.
759 for (const auto& picture_buffer_at_display : picture_buffers_at_display_)
760 assigned_picture_buffers_.erase(picture_buffer_at_display.first);
761
762 for (const auto& assigned_picture_buffer : assigned_picture_buffers_)
763 factories_->DeleteTexture(assigned_picture_buffer.second.texture_ids()[0]);
764
765 assigned_picture_buffers_.clear();
766 }
767
768 void RTCVideoDecoder::DestroyVDA() {
769 DVLOG(2) << "DestroyVDA";
770 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
771 if (vda_)
772 vda_.release()->Destroy();
773 DestroyTextures();
774
775 base::AutoLock auto_lock(lock_);
776
777 // Put the buffers back in case we restart the decoder.
778 for (auto& buffer : bitstream_buffers_in_decoder_)
779 PutSHM_Locked(std::move(buffer.second));
780 bitstream_buffers_in_decoder_.clear();
781
782 state_ = UNINITIALIZED;
783 }
784
785 std::unique_ptr<base::SharedMemory> RTCVideoDecoder::GetSHM_Locked(
786 size_t min_size) {
787 // Reuse a SHM if possible.
788 if (!available_shm_segments_.empty() &&
789 available_shm_segments_.back()->mapped_size() >= min_size) {
790 std::unique_ptr<base::SharedMemory> buffer =
791 std::move(available_shm_segments_.back());
792 available_shm_segments_.pop_back();
793 return buffer;
794 }
795
796 if (available_shm_segments_.size() != num_shm_buffers_) {
797 // Either available_shm_segments_ is empty (and we already have some SHM
798 // buffers allocated), or the size of available segments is not large
799 // enough. In the former case we need to wait for buffers to be returned,
800 // in the latter we need to wait for all buffers to be returned to drop
801 // them and reallocate with a new size.
802 return NULL;
803 }
804
805 if (num_shm_buffers_ != 0) {
806 available_shm_segments_.clear();
807 num_shm_buffers_ = 0;
808 }
809
810 // Create twice as large buffers as required, to avoid frequent reallocation.
811 factories_->GetTaskRunner()->PostTask(
812 FROM_HERE,
813 base::Bind(&RTCVideoDecoder::CreateSHM, weak_factory_.GetWeakPtr(),
814 kNumSharedMemorySegments, min_size * 2));
815
816 // We'll be called again after the shared memory is created.
817 return NULL;
818 }
819
820 void RTCVideoDecoder::PutSHM_Locked(
821 std::unique_ptr<base::SharedMemory> shm_buffer) {
822 lock_.AssertAcquired();
823 available_shm_segments_.push_back(std::move(shm_buffer));
824 }
825
826 void RTCVideoDecoder::CreateSHM(size_t count, size_t size) {
827 DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
828 DVLOG(2) << "CreateSHM. count=" << count << ", size=" << size;
829
830 for (size_t i = 0; i < count; i++) {
831 std::unique_ptr<base::SharedMemory> shm =
832 factories_->CreateSharedMemory(size);
833 if (!shm) {
834 LOG(ERROR) << "Failed allocating shared memory of size=" << size;
835 NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
836 return;
837 }
838
839 base::AutoLock auto_lock(lock_); 572 base::AutoLock auto_lock(lock_);
840 PutSHM_Locked(std::move(shm)); 573 DCHECK(decode_complete_callback_);
841 ++num_shm_buffers_; 574 decode_complete_callback_->Decoded(decoded_image);
842 } 575
843 576 // Reset error counter as we successfully decoded a frame.
844 // Kick off the decoding. 577 decoder_error_counter_ = 0;
845 RequestBufferDecode(); 578 }
846 }
847
848 void RTCVideoDecoder::RecordBufferData(const BufferData& buffer_data) {
849 input_buffer_data_.push_front(buffer_data);
850 // Why this value? Because why not. avformat.h:MAX_REORDER_DELAY is 16, but
851 // that's too small for some pathological B-frame test videos. The cost of
852 // using too-high a value is low (192 bits per extra slot).
853 static const size_t kMaxInputBufferDataSize = 128;
854 // Pop from the back of the list, because that's the oldest and least likely
855 // to be useful in the future data.
856 if (input_buffer_data_.size() > kMaxInputBufferDataSize)
857 input_buffer_data_.pop_back();
858 }
859
860 void RTCVideoDecoder::GetBufferData(int32_t bitstream_buffer_id,
861 uint32_t* timestamp,
862 gfx::Rect* visible_rect) {
863 for (const auto& buffer_data : input_buffer_data_) {
864 if (buffer_data.bitstream_buffer_id != bitstream_buffer_id)
865 continue;
866 *timestamp = buffer_data.timestamp;
867 *visible_rect = buffer_data.visible_rect;
868 return;
869 }
870 NOTREACHED() << "Missing bitstream buffer id: " << bitstream_buffer_id;
871 } 579 }
872 580
873 int32_t RTCVideoDecoder::RecordInitDecodeUMA(int32_t status) { 581 int32_t RTCVideoDecoder::RecordInitDecodeUMA(int32_t status) {
582 DVLOG(2) << __func__;
583
874 // Logging boolean is enough to know if HW decoding has been used. Also, 584 // Logging boolean is enough to know if HW decoding has been used. Also,
875 // InitDecode is less likely to return an error so enum is not used here. 585 // InitDecode is less likely to return an error so enum is not used here.
876 bool sample = (status == WEBRTC_VIDEO_CODEC_OK) ? true : false; 586 bool sample = (status == WEBRTC_VIDEO_CODEC_OK) ? true : false;
877 UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoDecoderInitDecodeSuccess", sample); 587 UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoDecoderInitDecodeSuccess", sample);
878 return status; 588 return status;
879 } 589 }
880 590
881 void RTCVideoDecoder::DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent() 591 void RTCVideoDecoder::ClearPendingBuffers_Locked() {
882 const { 592 DVLOG(2) << __func__;
883 DCHECK(factories_->GetTaskRunner()->BelongsToCurrentThread());
884 }
885 593
886 void RTCVideoDecoder::ClearPendingBuffers() { 594 lock_.AssertAcquired();
887 // Delete WebRTC input buffers.
888 for (const auto& pending_buffer : pending_buffers_)
889 delete[] pending_buffer.first._buffer;
890 pending_buffers_.clear(); 595 pending_buffers_.clear();
891 } 596 }
892 597
893 } // namespace content 598 } // namespace content
OLDNEW
« no previous file with comments | « content/renderer/media/gpu/rtc_video_decoder.h ('k') | content/renderer/media/gpu/rtc_video_decoder_factory.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698