Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(46)

Side by Side Diff: content/renderer/media_recorder/video_track_recorder.cc

Issue 2793303003: Refactor VideoTrackRecorder into smaller classes (Closed)
Patch Set: Rebase Created 3 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2015 The Chromium Authors. All rights reserved. 1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "content/renderer/media_recorder/video_track_recorder.h" 5 #include "content/renderer/media_recorder/video_track_recorder.h"
6 6
7 #include <utility> 7 #include <utility>
8 8
9 #include "base/bind.h" 9 #include "base/bind.h"
10 #include "base/logging.h" 10 #include "base/logging.h"
11 #include "base/macros.h" 11 #include "base/macros.h"
12 #include "base/memory/ptr_util.h" 12 #include "base/memory/ptr_util.h"
13 #include "base/sys_info.h"
14 #include "base/task_runner_util.h" 13 #include "base/task_runner_util.h"
15 #include "base/threading/thread.h" 14 #include "base/threading/thread.h"
16 #include "base/threading/thread_task_runner_handle.h" 15 #include "base/threading/thread_task_runner_handle.h"
17 #include "base/time/time.h" 16 #include "base/time/time.h"
18 #include "base/trace_event/trace_event.h"
19 #include "cc/paint/skia_paint_canvas.h" 17 #include "cc/paint/skia_paint_canvas.h"
20 #include "content/renderer/media/renderer_gpu_video_accelerator_factories.h" 18 #include "content/renderer/media/renderer_gpu_video_accelerator_factories.h"
19 #include "content/renderer/media_recorder/vea_encoder.h"
20 #include "content/renderer/media_recorder/vpx_encoder.h"
21 #include "content/renderer/render_thread_impl.h" 21 #include "content/renderer/render_thread_impl.h"
22 #include "media/base/bind_to_current_loop.h" 22 #include "media/base/bind_to_current_loop.h"
23 #include "media/base/video_frame.h" 23 #include "media/base/video_frame.h"
24 #include "media/base/video_util.h" 24 #include "media/base/video_util.h"
25 #include "media/filters/context_3d.h" 25 #include "media/filters/context_3d.h"
26 #include "media/renderers/skcanvas_video_renderer.h" 26 #include "media/renderers/skcanvas_video_renderer.h"
27 #include "services/ui/public/cpp/gpu/context_provider_command_buffer.h" 27 #include "services/ui/public/cpp/gpu/context_provider_command_buffer.h"
28 #include "skia/ext/platform_canvas.h" 28 #include "skia/ext/platform_canvas.h"
29 #include "third_party/libyuv/include/libyuv.h" 29 #include "third_party/libyuv/include/libyuv.h"
30 #include "ui/gfx/geometry/size.h" 30 #include "ui/gfx/geometry/size.h"
31 31
32 #if BUILDFLAG(RTC_USE_H264) 32 #if BUILDFLAG(RTC_USE_H264)
33 #include "third_party/openh264/src/codec/api/svc/codec_api.h" 33 #include "content/renderer/media_recorder/h264_encoder.h"
34 #include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
35 #include "third_party/openh264/src/codec/api/svc/codec_def.h"
36 #endif // #if BUILDFLAG(RTC_USE_H264) 34 #endif // #if BUILDFLAG(RTC_USE_H264)
37 35
38 extern "C" {
39 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
40 // backwards compatibility for legacy applications using the library.
41 #define VPX_CODEC_DISABLE_COMPAT 1
42 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
43 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
44 }
45
46 using media::VideoFrame; 36 using media::VideoFrame;
47 using media::VideoFrameMetadata;
48 using video_track_recorder::kVEAEncoderMinResolutionWidth; 37 using video_track_recorder::kVEAEncoderMinResolutionWidth;
49 using video_track_recorder::kVEAEncoderMinResolutionHeight; 38 using video_track_recorder::kVEAEncoderMinResolutionHeight;
50 39
51 namespace content { 40 namespace content {
52 41
53 namespace { 42 namespace {
54 43
55 // HW encoders expect a nonzero bitrate, so |kVEADefaultBitratePerPixel| is used
56 // to estimate bits per second for ~30 fps with ~1/16 compression rate.
57 const int kVEADefaultBitratePerPixel = 2;
58 // Number of output buffers used to copy the encoded data coming from HW
59 // encoders.
60 const int kVEAEncoderOutputBufferCount = 4;
61
62 using CodecId = VideoTrackRecorder::CodecId; 44 using CodecId = VideoTrackRecorder::CodecId;
63 45
64 static const struct { 46 static const struct {
65 CodecId codec_id; 47 CodecId codec_id;
66 media::VideoCodecProfile min_profile; 48 media::VideoCodecProfile min_profile;
67 media::VideoCodecProfile max_profile; 49 media::VideoCodecProfile max_profile;
68 } kPreferredCodecIdAndVEAProfiles[] = { 50 } kPreferredCodecIdAndVEAProfiles[] = {
69 {CodecId::VP8, media::VP8PROFILE_MIN, media::VP8PROFILE_MAX}, 51 {CodecId::VP8, media::VP8PROFILE_MIN, media::VP8PROFILE_MAX},
70 {CodecId::VP9, media::VP9PROFILE_MIN, media::VP9PROFILE_MAX}, 52 {CodecId::VP9, media::VP9PROFILE_MIN, media::VP9PROFILE_MAX},
71 #if BUILDFLAG(RTC_USE_H264) 53 #if BUILDFLAG(RTC_USE_H264)
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after
153 135
154 media::VideoCodecProfile CodecEnumerator::CodecIdToVEAProfile(CodecId codec) { 136 media::VideoCodecProfile CodecEnumerator::CodecIdToVEAProfile(CodecId codec) {
155 const auto profile = codec_id_to_profile_.find(codec); 137 const auto profile = codec_id_to_profile_.find(codec);
156 return profile == codec_id_to_profile_.end() 138 return profile == codec_id_to_profile_.end()
157 ? media::VIDEO_CODEC_PROFILE_UNKNOWN 139 ? media::VIDEO_CODEC_PROFILE_UNKNOWN
158 : profile->second; 140 : profile->second;
159 } 141 }
160 142
161 } // anonymous namespace 143 } // anonymous namespace
162 144
163 // Base class to describe a generic Encoder, encapsulating all actual encoder 145 VideoTrackRecorder::Encoder::Encoder(
164 // (re)configurations, encoding and delivery of received frames. This class is 146 const OnEncodedVideoCB& on_encoded_video_callback,
165 // ref-counted to allow the MediaStreamVideoTrack to hold a reference to it (via 147 int32_t bits_per_second,
166 // the callback that MediaStreamVideoSink passes along) and to jump back and 148 scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner)
167 // forth to an internal encoder thread. Moreover, this class: 149 : main_task_runner_(base::ThreadTaskRunnerHandle::Get()),
168 // - is created on its parent's thread (usually the main Render thread), 150 encoding_task_runner_(encoding_task_runner),
169 // that is, |main_task_runner_|. 151 paused_(false),
170 // - receives VideoFrames on |origin_task_runner_| and runs OnEncodedVideoCB on 152 on_encoded_video_callback_(on_encoded_video_callback),
171 // that thread as well. This task runner is cached on first frame arrival, and 153 bits_per_second_(bits_per_second) {
172 // is supposed to be the render IO thread (but this is not enforced); 154 DCHECK(!on_encoded_video_callback_.is_null());
173 // - uses an internal |encoding_task_runner_| for actual encoder interactions, 155 if (encoding_task_runner_)
174 // namely configuration, encoding (which might take some time) and destruction. 156 return;
175 // This task runner can be passed on the creation. If nothing is passed, a new 157 encoding_thread_.reset(new base::Thread("EncodingThread"));
176 // encoding thread is created and used. 158 encoding_thread_->Start();
177 class VideoTrackRecorder::Encoder : public base::RefCountedThreadSafe<Encoder> { 159 encoding_task_runner_ = encoding_thread_->task_runner();
178 public: 160 }
179 Encoder(const OnEncodedVideoCB& on_encoded_video_callback,
180 int32_t bits_per_second,
181 scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner =
182 nullptr)
183 : main_task_runner_(base::ThreadTaskRunnerHandle::Get()),
184 encoding_task_runner_(encoding_task_runner),
185 paused_(false),
186 on_encoded_video_callback_(on_encoded_video_callback),
187 bits_per_second_(bits_per_second) {
188 DCHECK(!on_encoded_video_callback_.is_null());
189 if (encoding_task_runner_)
190 return;
191 encoding_thread_.reset(new base::Thread("EncodingThread"));
192 encoding_thread_->Start();
193 encoding_task_runner_ = encoding_thread_->task_runner();
194 }
195 161
196 // Start encoding |frame|, returning via |on_encoded_video_callback_|. This 162 VideoTrackRecorder::Encoder::~Encoder() {
197 // call will also trigger an encode configuration upon first frame arrival 163 main_task_runner_->DeleteSoon(FROM_HERE, video_renderer_.release());
198 // or parameter change, and an EncodeOnEncodingTaskRunner() to actually 164 }
199 // encode the frame. If the |frame|'s data is not directly available (e.g.
200 // it's a texture) then RetrieveFrameOnMainThread() is called, and if even
201 // that fails, black frames are sent instead.
202 void StartFrameEncode(const scoped_refptr<VideoFrame>& frame,
203 base::TimeTicks capture_timestamp);
204 void RetrieveFrameOnMainThread(const scoped_refptr<VideoFrame>& video_frame,
205 base::TimeTicks capture_timestamp);
206
207 void SetPaused(bool paused);
208 virtual bool CanEncodeAlphaChannel() { return false; }
209
210 protected:
211 friend class base::RefCountedThreadSafe<Encoder>;
212 virtual ~Encoder() {
213 main_task_runner_->DeleteSoon(FROM_HERE, video_renderer_.release());
214 }
215
216 virtual void EncodeOnEncodingTaskRunner(
217 scoped_refptr<VideoFrame> frame,
218 base::TimeTicks capture_timestamp) = 0;
219
220 // Used to shutdown properly on the same thread we were created.
221 const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
222
223 // Task runner where frames to encode and reply callbacks must happen.
224 scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
225
226 // Task runner where encoding interactions happen.
227 scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner_;
228
229 // Optional thread for encoding. Active for the lifetime of VpxEncoder.
230 std::unique_ptr<base::Thread> encoding_thread_;
231
232 // While |paused_|, frames are not encoded. Used only from |encoding_thread_|.
233 bool paused_;
234
235 // This callback should be exercised on IO thread.
236 const OnEncodedVideoCB on_encoded_video_callback_;
237
238 // Target bitrate for video encoding. If 0, a standard bitrate is used.
239 const int32_t bits_per_second_;
240
241 // Used to retrieve incoming opaque VideoFrames (i.e. VideoFrames backed by
242 // textures). Created on-demand on |main_task_runner_|.
243 std::unique_ptr<media::SkCanvasVideoRenderer> video_renderer_;
244 SkBitmap bitmap_;
245 std::unique_ptr<cc::PaintCanvas> canvas_;
246
247 DISALLOW_COPY_AND_ASSIGN(Encoder);
248 };
249 165
250 void VideoTrackRecorder::Encoder::StartFrameEncode( 166 void VideoTrackRecorder::Encoder::StartFrameEncode(
251 const scoped_refptr<VideoFrame>& video_frame, 167 const scoped_refptr<VideoFrame>& video_frame,
252 base::TimeTicks capture_timestamp) { 168 base::TimeTicks capture_timestamp) {
253 // Cache the thread sending frames on first frame arrival. 169 // Cache the thread sending frames on first frame arrival.
254 if (!origin_task_runner_.get()) 170 if (!origin_task_runner_.get())
255 origin_task_runner_ = base::ThreadTaskRunnerHandle::Get(); 171 origin_task_runner_ = base::ThreadTaskRunnerHandle::Get();
256 DCHECK(origin_task_runner_->BelongsToCurrentThread()); 172 DCHECK(origin_task_runner_->BelongsToCurrentThread());
257 if (paused_) 173 if (paused_)
258 return; 174 return;
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after
354 DLOG(ERROR) << "Error converting frame to I420"; 270 DLOG(ERROR) << "Error converting frame to I420";
355 return; 271 return;
356 } 272 }
357 } 273 }
358 274
359 encoding_task_runner_->PostTask( 275 encoding_task_runner_->PostTask(
360 FROM_HERE, base::Bind(&Encoder::EncodeOnEncodingTaskRunner, this, frame, 276 FROM_HERE, base::Bind(&Encoder::EncodeOnEncodingTaskRunner, this, frame,
361 capture_timestamp)); 277 capture_timestamp));
362 } 278 }
363 279
364 void VideoTrackRecorder::Encoder::SetPaused(bool paused) { 280 // static
365 if (!encoding_task_runner_->BelongsToCurrentThread()) { 281 void VideoTrackRecorder::Encoder::OnFrameEncodeCompleted(
366 encoding_task_runner_->PostTask(
367 FROM_HERE, base::Bind(&Encoder::SetPaused, this, paused));
368 return;
369 }
370 paused_ = paused;
371 }
372
373 namespace {
374
375 // Originally from remoting/codec/scoped_vpx_codec.h.
376 // TODO(mcasas): Refactor into a common location.
377 struct VpxCodecDeleter {
378 void operator()(vpx_codec_ctx_t* codec) {
379 if (!codec)
380 return;
381 vpx_codec_err_t ret = vpx_codec_destroy(codec);
382 CHECK_EQ(ret, VPX_CODEC_OK);
383 delete codec;
384 }
385 };
386 typedef std::unique_ptr<vpx_codec_ctx_t, VpxCodecDeleter> ScopedVpxCodecCtxPtr;
387
388 static void OnFrameEncodeCompleted(
389 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb, 282 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
390 const media::WebmMuxer::VideoParameters& params, 283 const media::WebmMuxer::VideoParameters& params,
391 std::unique_ptr<std::string> data, 284 std::unique_ptr<std::string> data,
392 std::unique_ptr<std::string> alpha_data, 285 std::unique_ptr<std::string> alpha_data,
393 base::TimeTicks capture_timestamp, 286 base::TimeTicks capture_timestamp,
394 bool keyframe) { 287 bool keyframe) {
395 DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "<< data->length() << "B, " 288 DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "<< data->length() << "B, "
396 << capture_timestamp << " ms"; 289 << capture_timestamp << " ms";
397 on_encoded_video_cb.Run(params, std::move(data), std::move(alpha_data), 290 on_encoded_video_cb.Run(params, std::move(data), std::move(alpha_data),
398 capture_timestamp, keyframe); 291 capture_timestamp, keyframe);
399 } 292 }
400 293
401 static int GetNumberOfThreadsForEncoding() { 294 void VideoTrackRecorder::Encoder::SetPaused(bool paused) {
402 // Do not saturate CPU utilization just for encoding. On a lower-end system 295 if (!encoding_task_runner_->BelongsToCurrentThread()) {
403 // with only 1 or 2 cores, use only one thread for encoding. On systems with 296 encoding_task_runner_->PostTask(
404 // more cores, allow half of the cores to be used for encoding. 297 FROM_HERE, base::Bind(&Encoder::SetPaused, this, paused));
405 return std::min(8, (base::SysInfo::NumberOfProcessors() + 1) / 2); 298 return;
299 }
300 paused_ = paused;
406 } 301 }
407 302
408 // Class encapsulating VideoEncodeAccelerator interactions. 303 bool VideoTrackRecorder::Encoder::CanEncodeAlphaChannel() {
409 // This class is created and destroyed in its owner thread. All other methods 304 return false;
410 // operate on the task runner pointed by GpuFactories.
411 class VEAEncoder final : public VideoTrackRecorder::Encoder,
412 public media::VideoEncodeAccelerator::Client {
413 public:
414 VEAEncoder(
415 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
416 const VideoTrackRecorder::OnErrorCB& on_error_callback,
417 int32_t bits_per_second,
418 media::VideoCodecProfile codec,
419 const gfx::Size& size);
420
421 // media::VideoEncodeAccelerator::Client implementation.
422 void RequireBitstreamBuffers(unsigned int input_count,
423 const gfx::Size& input_coded_size,
424 size_t output_buffer_size) override;
425 void BitstreamBufferReady(int32_t bitstream_buffer_id,
426 size_t payload_size,
427 bool key_frame,
428 base::TimeDelta timestamp) override;
429 void NotifyError(media::VideoEncodeAccelerator::Error error) override;
430
431 private:
432 using VideoFrameAndTimestamp =
433 std::pair<scoped_refptr<media::VideoFrame>, base::TimeTicks>;
434 using VideoParamsAndTimestamp =
435 std::pair<media::WebmMuxer::VideoParameters, base::TimeTicks>;
436
437 void UseOutputBitstreamBufferId(int32_t bitstream_buffer_id);
438 void FrameFinished(std::unique_ptr<base::SharedMemory> shm);
439
440 // VideoTrackRecorder::Encoder implementation.
441 ~VEAEncoder() override;
442 void EncodeOnEncodingTaskRunner(scoped_refptr<VideoFrame> frame,
443 base::TimeTicks capture_timestamp) override;
444
445 void ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size);
446
447 void DestroyOnEncodingTaskRunner(base::WaitableEvent* async_waiter);
448
449 media::GpuVideoAcceleratorFactories* const gpu_factories_;
450
451 const media::VideoCodecProfile codec_;
452
453 // The underlying VEA to perform encoding on.
454 std::unique_ptr<media::VideoEncodeAccelerator> video_encoder_;
455
456 // Shared memory buffers for output with the VEA.
457 std::vector<std::unique_ptr<base::SharedMemory>> output_buffers_;
458
459 // Shared memory buffers for output with the VEA as FIFO.
460 std::queue<std::unique_ptr<base::SharedMemory>> input_buffers_;
461
462 // Tracks error status.
463 bool error_notified_;
464
465 // Tracks the last frame that we delay the encode.
466 std::unique_ptr<VideoFrameAndTimestamp> last_frame_;
467
468 // Size used to initialize encoder.
469 gfx::Size input_visible_size_;
470
471 // Coded size that encoder requests as input.
472 gfx::Size vea_requested_input_coded_size_;
473
474 // Frames and corresponding timestamps in encode as FIFO.
475 std::queue<VideoParamsAndTimestamp> frames_in_encode_;
476
477 // This callback can be exercised on any thread.
478 const VideoTrackRecorder::OnErrorCB on_error_callback_;
479 };
480
481 // Class encapsulating all libvpx interactions for VP8/VP9 encoding.
482 class VpxEncoder final : public VideoTrackRecorder::Encoder {
483 public:
484 static void ShutdownEncoder(std::unique_ptr<base::Thread> encoding_thread,
485 ScopedVpxCodecCtxPtr encoder);
486
487 VpxEncoder(
488 bool use_vp9,
489 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
490 int32_t bits_per_second);
491
492 private:
493 // VideoTrackRecorder::Encoder implementation.
494 ~VpxEncoder() override;
495 void EncodeOnEncodingTaskRunner(scoped_refptr<VideoFrame> frame,
496 base::TimeTicks capture_timestamp) override;
497 bool CanEncodeAlphaChannel() override { return true; }
498
499 void ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size,
500 vpx_codec_enc_cfg_t* codec_config,
501 ScopedVpxCodecCtxPtr* encoder);
502 void DoEncode(vpx_codec_ctx_t* const encoder,
503 const gfx::Size& frame_size,
504 uint8_t* const data,
505 uint8_t* const y_plane,
506 int y_stride,
507 uint8_t* const u_plane,
508 int u_stride,
509 uint8_t* const v_plane,
510 int v_stride,
511 const base::TimeDelta& duration,
512 bool force_keyframe,
513 std::string* const output_data,
514 bool* const keyframe);
515
516 // Returns true if |codec_config| has been filled in at least once.
517 bool IsInitialized(const vpx_codec_enc_cfg_t& codec_config) const;
518
519 // Estimate the frame duration from |frame| and |last_frame_timestamp_|.
520 base::TimeDelta EstimateFrameDuration(const scoped_refptr<VideoFrame>& frame);
521
522 // Force usage of VP9 for encoding, instead of VP8 which is the default.
523 const bool use_vp9_;
524
525 // VPx internal objects: configuration and encoder. |encoder_| is a special
526 // scoped pointer to guarantee proper destruction, particularly when
527 // reconfiguring due to parameters change. Only used on |encoding_thread_|.
528 vpx_codec_enc_cfg_t codec_config_;
529 ScopedVpxCodecCtxPtr encoder_;
530
531 vpx_codec_enc_cfg_t alpha_codec_config_;
532 ScopedVpxCodecCtxPtr alpha_encoder_;
533
534 std::vector<uint8_t> alpha_dummy_planes_;
535 size_t v_plane_offset_;
536 size_t u_plane_stride_;
537 size_t v_plane_stride_;
538 bool last_frame_had_alpha_ = false;
539
540 // The |VideoFrame::timestamp()| of the last encoded frame. This is used to
541 // predict the duration of the next frame. Only used on |encoding_thread_|.
542 base::TimeDelta last_frame_timestamp_;
543
544 DISALLOW_COPY_AND_ASSIGN(VpxEncoder);
545 };
546
547 #if BUILDFLAG(RTC_USE_H264)
548
549 struct ISVCEncoderDeleter {
550 void operator()(ISVCEncoder* codec) {
551 if (!codec)
552 return;
553 const int uninit_ret = codec->Uninitialize();
554 CHECK_EQ(cmResultSuccess, uninit_ret);
555 WelsDestroySVCEncoder(codec);
556 }
557 };
558 typedef std::unique_ptr<ISVCEncoder, ISVCEncoderDeleter> ScopedISVCEncoderPtr;
559
560 // Class encapsulating all openh264 interactions for H264 encoding.
561 class H264Encoder final : public VideoTrackRecorder::Encoder {
562 public:
563 static void ShutdownEncoder(std::unique_ptr<base::Thread> encoding_thread,
564 ScopedISVCEncoderPtr encoder);
565
566 H264Encoder(
567 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
568 int32_t bits_per_second);
569
570 private:
571 // VideoTrackRecorder::Encoder implementation.
572 ~H264Encoder() override;
573 void EncodeOnEncodingTaskRunner(scoped_refptr<VideoFrame> frame,
574 base::TimeTicks capture_timestamp) override;
575
576 void ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size);
577
578 // |openh264_encoder_| is a special scoped pointer to guarantee proper
579 // destruction, also when reconfiguring due to parameters change. Only used on
580 // |encoding_thread_|.
581 gfx::Size configured_size_;
582 ScopedISVCEncoderPtr openh264_encoder_;
583
584 // The |VideoFrame::timestamp()| of the first received frame. Only used on
585 // |encoding_thread_|.
586 base::TimeTicks first_frame_timestamp_;
587
588 DISALLOW_COPY_AND_ASSIGN(H264Encoder);
589 };
590
591 #endif // #if BUILDFLAG(RTC_USE_H264)
592
593 VEAEncoder::VEAEncoder(
594 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
595 const VideoTrackRecorder::OnErrorCB& on_error_callback,
596 int32_t bits_per_second,
597 media::VideoCodecProfile codec,
598 const gfx::Size& size)
599 : Encoder(on_encoded_video_callback,
600 bits_per_second > 0 ? bits_per_second
601 : size.GetArea() * kVEADefaultBitratePerPixel,
602 RenderThreadImpl::current()->GetGpuFactories()->GetTaskRunner()),
603 gpu_factories_(RenderThreadImpl::current()->GetGpuFactories()),
604 codec_(codec),
605 error_notified_(false),
606 on_error_callback_(on_error_callback) {
607 DCHECK(gpu_factories_);
608 DCHECK_GE(size.width(), kVEAEncoderMinResolutionWidth);
609 DCHECK_GE(size.height(), kVEAEncoderMinResolutionHeight);
610
611 encoding_task_runner_->PostTask(
612 FROM_HERE, base::Bind(&VEAEncoder::ConfigureEncoderOnEncodingTaskRunner,
613 this, size));
614 }
615
616 VEAEncoder::~VEAEncoder() {
617 base::WaitableEvent release_waiter(
618 base::WaitableEvent::ResetPolicy::MANUAL,
619 base::WaitableEvent::InitialState::NOT_SIGNALED);
620 // base::Unretained is safe because the class will be alive until
621 // |release_waiter| is signaled.
622 // TODO(emircan): Consider refactoring media::VideoEncodeAccelerator to avoid
623 // using naked pointers and using DeleteSoon() here, see
624 // http://crbug.com/701627.
625 // It is currently unsafe because |video_encoder_| might be in use on another
626 // function on |encoding_task_runner_|, see http://crbug.com/701030.
627 encoding_task_runner_->PostTask(
628 FROM_HERE, base::Bind(&VEAEncoder::DestroyOnEncodingTaskRunner,
629 base::Unretained(this), &release_waiter));
630 release_waiter.Wait();
631 }
632
633 void VEAEncoder::RequireBitstreamBuffers(unsigned int /*input_count*/,
634 const gfx::Size& input_coded_size,
635 size_t output_buffer_size) {
636 DVLOG(3) << __func__;
637 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
638
639 vea_requested_input_coded_size_ = input_coded_size;
640 output_buffers_.clear();
641 std::queue<std::unique_ptr<base::SharedMemory>>().swap(input_buffers_);
642
643 for (int i = 0; i < kVEAEncoderOutputBufferCount; ++i) {
644 std::unique_ptr<base::SharedMemory> shm =
645 gpu_factories_->CreateSharedMemory(output_buffer_size);
646 if (shm)
647 output_buffers_.push_back(base::WrapUnique(shm.release()));
648 }
649
650 for (size_t i = 0; i < output_buffers_.size(); ++i)
651 UseOutputBitstreamBufferId(i);
652 }
653
654 void VEAEncoder::BitstreamBufferReady(int32_t bitstream_buffer_id,
655 size_t payload_size,
656 bool keyframe,
657 base::TimeDelta timestamp) {
658 DVLOG(3) << __func__;
659 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
660
661 base::SharedMemory* output_buffer =
662 output_buffers_[bitstream_buffer_id].get();
663
664 std::unique_ptr<std::string> data(new std::string);
665 data->append(reinterpret_cast<char*>(output_buffer->memory()), payload_size);
666
667 const auto front_frame = frames_in_encode_.front();
668 frames_in_encode_.pop();
669 origin_task_runner_->PostTask(
670 FROM_HERE, base::Bind(OnFrameEncodeCompleted, on_encoded_video_callback_,
671 front_frame.first, base::Passed(&data), nullptr,
672 front_frame.second, keyframe));
673 UseOutputBitstreamBufferId(bitstream_buffer_id);
674 }
675
676 void VEAEncoder::NotifyError(media::VideoEncodeAccelerator::Error error) {
677 DVLOG(3) << __func__;
678 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
679 on_error_callback_.Run();
680 error_notified_ = true;
681 }
682
683 void VEAEncoder::UseOutputBitstreamBufferId(int32_t bitstream_buffer_id) {
684 DVLOG(3) << __func__;
685 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
686
687 video_encoder_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
688 bitstream_buffer_id, output_buffers_[bitstream_buffer_id]->handle(),
689 output_buffers_[bitstream_buffer_id]->mapped_size()));
690 }
691
692 void VEAEncoder::FrameFinished(std::unique_ptr<base::SharedMemory> shm) {
693 DVLOG(3) << __func__;
694 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
695 input_buffers_.push(std::move(shm));
696 }
697
698 void VEAEncoder::EncodeOnEncodingTaskRunner(
699 scoped_refptr<VideoFrame> frame,
700 base::TimeTicks capture_timestamp) {
701 DVLOG(3) << __func__;
702 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
703
704 if (input_visible_size_ != frame->visible_rect().size() && video_encoder_)
705 video_encoder_.reset();
706
707 if (!video_encoder_)
708 ConfigureEncoderOnEncodingTaskRunner(frame->visible_rect().size());
709
710 if (error_notified_) {
711 DVLOG(3) << "An error occurred in VEA encoder";
712 return;
713 }
714
715 // Drop frames if there is no output buffers available.
716 if (output_buffers_.empty()) {
717 // TODO(emircan): Investigate if resetting encoder would help.
718 DVLOG(3) << "Might drop frame.";
719 last_frame_.reset(
720 new std::pair<scoped_refptr<VideoFrame>, base::TimeTicks>(
721 frame, capture_timestamp));
722 return;
723 }
724
725 // If first frame hasn't been encoded, do it first.
726 if (last_frame_) {
727 std::unique_ptr<VideoFrameAndTimestamp> last_frame(last_frame_.release());
728 EncodeOnEncodingTaskRunner(last_frame->first, last_frame->second);
729 }
730
731 // Lower resolutions may fall back to SW encoder in some platforms, i.e. Mac.
732 // In that case, the encoder expects more frames before returning result.
733 // Therefore, a copy is necessary to release the current frame.
734 // Only STORAGE_SHMEM backed frames can be shared with GPU process, therefore
735 // a copy is required for other storage types.
736 scoped_refptr<media::VideoFrame> video_frame = frame;
737 if (video_frame->storage_type() != VideoFrame::STORAGE_SHMEM ||
738 vea_requested_input_coded_size_ != frame->coded_size() ||
739 input_visible_size_.width() < kVEAEncoderMinResolutionWidth ||
740 input_visible_size_.height() < kVEAEncoderMinResolutionHeight) {
741 // Create SharedMemory backed input buffers as necessary. These SharedMemory
742 // instances will be shared with GPU process.
743 std::unique_ptr<base::SharedMemory> input_buffer;
744 const size_t desired_mapped_size = media::VideoFrame::AllocationSize(
745 media::PIXEL_FORMAT_I420, vea_requested_input_coded_size_);
746 if (input_buffers_.empty()) {
747 input_buffer = gpu_factories_->CreateSharedMemory(desired_mapped_size);
748 } else {
749 do {
750 input_buffer = std::move(input_buffers_.front());
751 input_buffers_.pop();
752 } while (!input_buffers_.empty() &&
753 input_buffer->mapped_size() < desired_mapped_size);
754 if (!input_buffer || input_buffer->mapped_size() < desired_mapped_size)
755 return;
756 }
757
758 video_frame = media::VideoFrame::WrapExternalSharedMemory(
759 media::PIXEL_FORMAT_I420, vea_requested_input_coded_size_,
760 gfx::Rect(input_visible_size_), input_visible_size_,
761 reinterpret_cast<uint8_t*>(input_buffer->memory()),
762 input_buffer->mapped_size(), input_buffer->handle(), 0,
763 frame->timestamp());
764 video_frame->AddDestructionObserver(media::BindToCurrentLoop(
765 base::Bind(&VEAEncoder::FrameFinished, this,
766 base::Passed(std::move(input_buffer)))));
767 libyuv::I420Copy(frame->visible_data(media::VideoFrame::kYPlane),
768 frame->stride(media::VideoFrame::kYPlane),
769 frame->visible_data(media::VideoFrame::kUPlane),
770 frame->stride(media::VideoFrame::kUPlane),
771 frame->visible_data(media::VideoFrame::kVPlane),
772 frame->stride(media::VideoFrame::kVPlane),
773 video_frame->visible_data(media::VideoFrame::kYPlane),
774 video_frame->stride(media::VideoFrame::kYPlane),
775 video_frame->visible_data(media::VideoFrame::kUPlane),
776 video_frame->stride(media::VideoFrame::kUPlane),
777 video_frame->visible_data(media::VideoFrame::kVPlane),
778 video_frame->stride(media::VideoFrame::kVPlane),
779 input_visible_size_.width(), input_visible_size_.height());
780 }
781 frames_in_encode_.push(std::make_pair(
782 media::WebmMuxer::VideoParameters(frame), capture_timestamp));
783
784 video_encoder_->Encode(video_frame, false);
785 }
786
787 void VEAEncoder::ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) {
788 DVLOG(3) << __func__;
789 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
790 DCHECK(gpu_factories_->GetTaskRunner()->BelongsToCurrentThread());
791 DCHECK_GT(bits_per_second_, 0);
792
793 input_visible_size_ = size;
794 video_encoder_ = gpu_factories_->CreateVideoEncodeAccelerator();
795 if (!video_encoder_ ||
796 !video_encoder_->Initialize(media::PIXEL_FORMAT_I420, input_visible_size_,
797 codec_, bits_per_second_, this)) {
798 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
799 }
800 }
801
802 void VEAEncoder::DestroyOnEncodingTaskRunner(
803 base::WaitableEvent* async_waiter) {
804 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
805 video_encoder_.reset();
806 async_waiter->Signal();
807 } 305 }
808 306
809 // static 307 // static
810 void VpxEncoder::ShutdownEncoder(std::unique_ptr<base::Thread> encoding_thread,
811 ScopedVpxCodecCtxPtr encoder) {
812 DCHECK(encoding_thread->IsRunning());
813 encoding_thread->Stop();
814 // Both |encoding_thread| and |encoder| will be destroyed at end-of-scope.
815 }
816
817 VpxEncoder::VpxEncoder(
818 bool use_vp9,
819 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
820 int32_t bits_per_second)
821 : Encoder(on_encoded_video_callback, bits_per_second),
822 use_vp9_(use_vp9) {
823 codec_config_.g_timebase.den = 0; // Not initialized.
824 alpha_codec_config_.g_timebase.den = 0; // Not initialized.
825 DCHECK(encoding_thread_->IsRunning());
826 }
827
828 VpxEncoder::~VpxEncoder() {
829 main_task_runner_->PostTask(FROM_HERE,
830 base::Bind(&VpxEncoder::ShutdownEncoder,
831 base::Passed(&encoding_thread_),
832 base::Passed(&encoder_)));
833 }
834
835 void VpxEncoder::EncodeOnEncodingTaskRunner(
836 scoped_refptr<VideoFrame> frame,
837 base::TimeTicks capture_timestamp) {
838 TRACE_EVENT0("video", "VpxEncoder::EncodeOnEncodingTaskRunner");
839 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
840
841 const gfx::Size frame_size = frame->visible_rect().size();
842 const base::TimeDelta duration = EstimateFrameDuration(frame);
843 const media::WebmMuxer::VideoParameters video_params(frame);
844
845 if (!IsInitialized(codec_config_) ||
846 gfx::Size(codec_config_.g_w, codec_config_.g_h) != frame_size) {
847 ConfigureEncoderOnEncodingTaskRunner(frame_size, &codec_config_, &encoder_);
848 }
849
850 const bool frame_has_alpha = frame->format() == media::PIXEL_FORMAT_YV12A;
851 if (frame_has_alpha && (!IsInitialized(alpha_codec_config_) ||
852 gfx::Size(alpha_codec_config_.g_w,
853 alpha_codec_config_.g_h) != frame_size)) {
854 ConfigureEncoderOnEncodingTaskRunner(frame_size, &alpha_codec_config_,
855 &alpha_encoder_);
856 u_plane_stride_ = media::VideoFrame::RowBytes(
857 VideoFrame::kUPlane, frame->format(), frame_size.width());
858 v_plane_stride_ = media::VideoFrame::RowBytes(
859 VideoFrame::kVPlane, frame->format(), frame_size.width());
860 v_plane_offset_ = media::VideoFrame::PlaneSize(
861 frame->format(), VideoFrame::kUPlane, frame_size)
862 .GetArea();
863 alpha_dummy_planes_.resize(
864 v_plane_offset_ + media::VideoFrame::PlaneSize(
865 frame->format(), VideoFrame::kVPlane, frame_size)
866 .GetArea());
867 // It is more expensive to encode 0x00, so use 0x80 instead.
868 std::fill(alpha_dummy_planes_.begin(), alpha_dummy_planes_.end(), 0x80);
869 }
870 // If we introduced a new alpha frame, force keyframe.
871 const bool force_keyframe = frame_has_alpha && !last_frame_had_alpha_;
872 last_frame_had_alpha_ = frame_has_alpha;
873
874 std::unique_ptr<std::string> data(new std::string);
875 bool keyframe = false;
876 DoEncode(encoder_.get(), frame_size, frame->data(VideoFrame::kYPlane),
877 frame->visible_data(VideoFrame::kYPlane),
878 frame->stride(VideoFrame::kYPlane),
879 frame->visible_data(VideoFrame::kUPlane),
880 frame->stride(VideoFrame::kUPlane),
881 frame->visible_data(VideoFrame::kVPlane),
882 frame->stride(VideoFrame::kVPlane), duration, force_keyframe,
883 data.get(), &keyframe);
884
885 std::unique_ptr<std::string> alpha_data(new std::string);
886 if (frame_has_alpha) {
887 bool alpha_keyframe = false;
888 DoEncode(alpha_encoder_.get(), frame_size, frame->data(VideoFrame::kAPlane),
889 frame->visible_data(VideoFrame::kAPlane),
890 frame->stride(VideoFrame::kAPlane), alpha_dummy_planes_.data(),
891 u_plane_stride_, alpha_dummy_planes_.data() + v_plane_offset_,
892 v_plane_stride_, duration, keyframe, alpha_data.get(),
893 &alpha_keyframe);
894 DCHECK_EQ(keyframe, alpha_keyframe);
895 }
896 frame = nullptr;
897
898 origin_task_runner_->PostTask(
899 FROM_HERE,
900 base::Bind(OnFrameEncodeCompleted, on_encoded_video_callback_,
901 video_params, base::Passed(&data), base::Passed(&alpha_data),
902 capture_timestamp, keyframe));
903 }
904
905 void VpxEncoder::DoEncode(vpx_codec_ctx_t* const encoder,
906 const gfx::Size& frame_size,
907 uint8_t* const data,
908 uint8_t* const y_plane,
909 int y_stride,
910 uint8_t* const u_plane,
911 int u_stride,
912 uint8_t* const v_plane,
913 int v_stride,
914 const base::TimeDelta& duration,
915 bool force_keyframe,
916 std::string* const output_data,
917 bool* const keyframe) {
918 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
919
920 vpx_image_t vpx_image;
921 vpx_image_t* const result =
922 vpx_img_wrap(&vpx_image, VPX_IMG_FMT_I420, frame_size.width(),
923 frame_size.height(), 1 /* align */, data);
924 DCHECK_EQ(result, &vpx_image);
925 vpx_image.planes[VPX_PLANE_Y] = y_plane;
926 vpx_image.planes[VPX_PLANE_U] = u_plane;
927 vpx_image.planes[VPX_PLANE_V] = v_plane;
928 vpx_image.stride[VPX_PLANE_Y] = y_stride;
929 vpx_image.stride[VPX_PLANE_U] = u_stride;
930 vpx_image.stride[VPX_PLANE_V] = v_stride;
931
932 const vpx_codec_flags_t flags = force_keyframe ? VPX_EFLAG_FORCE_KF : 0;
933 // Encode the frame. The presentation time stamp argument here is fixed to
934 // zero to force the encoder to base its single-frame bandwidth calculations
935 // entirely on |predicted_frame_duration|.
936 const vpx_codec_err_t ret =
937 vpx_codec_encode(encoder, &vpx_image, 0 /* pts */,
938 duration.InMicroseconds(), flags, VPX_DL_REALTIME);
939 DCHECK_EQ(ret, VPX_CODEC_OK)
940 << vpx_codec_err_to_string(ret) << ", #" << vpx_codec_error(encoder)
941 << " -" << vpx_codec_error_detail(encoder);
942
943 *keyframe = false;
944 vpx_codec_iter_t iter = NULL;
945 const vpx_codec_cx_pkt_t* pkt = NULL;
946 while ((pkt = vpx_codec_get_cx_data(encoder, &iter)) != NULL) {
947 if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
948 continue;
949 output_data->assign(static_cast<char*>(pkt->data.frame.buf),
950 pkt->data.frame.sz);
951 *keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
952 break;
953 }
954 }
955
956 void VpxEncoder::ConfigureEncoderOnEncodingTaskRunner(
957 const gfx::Size& size,
958 vpx_codec_enc_cfg_t* codec_config,
959 ScopedVpxCodecCtxPtr* encoder) {
960 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
961 if (IsInitialized(*codec_config)) {
962 // TODO(mcasas) VP8 quirk/optimisation: If the new |size| is strictly less-
963 // than-or-equal than the old size, in terms of area, the existing encoder
964 // instance could be reused after changing |codec_config->{g_w,g_h}|.
965 DVLOG(1) << "Destroying/Re-Creating encoder for new frame size: "
966 << gfx::Size(codec_config->g_w, codec_config->g_h).ToString()
967 << " --> " << size.ToString() << (use_vp9_ ? " vp9" : " vp8");
968 encoder->reset();
969 }
970
971 const vpx_codec_iface_t* codec_interface =
972 use_vp9_ ? vpx_codec_vp9_cx() : vpx_codec_vp8_cx();
973 vpx_codec_err_t result = vpx_codec_enc_config_default(
974 codec_interface, codec_config, 0 /* reserved */);
975 DCHECK_EQ(VPX_CODEC_OK, result);
976
977 DCHECK_EQ(320u, codec_config->g_w);
978 DCHECK_EQ(240u, codec_config->g_h);
979 DCHECK_EQ(256u, codec_config->rc_target_bitrate);
980 // Use the selected bitrate or adjust default bit rate to account for the
981 // actual size. Note: |rc_target_bitrate| units are kbit per second.
982 if (bits_per_second_ > 0) {
983 codec_config->rc_target_bitrate = bits_per_second_ / 1000;
984 } else {
985 codec_config->rc_target_bitrate = size.GetArea() *
986 codec_config->rc_target_bitrate /
987 codec_config->g_w / codec_config->g_h;
988 }
989 // Both VP8/VP9 configuration should be Variable BitRate by default.
990 DCHECK_EQ(VPX_VBR, codec_config->rc_end_usage);
991 if (use_vp9_) {
992 // Number of frames to consume before producing output.
993 codec_config->g_lag_in_frames = 0;
994
995 // DCHECK that the profile selected by default is I420 (magic number 0).
996 DCHECK_EQ(0u, codec_config->g_profile);
997 } else {
998 // VP8 always produces frames instantaneously.
999 DCHECK_EQ(0u, codec_config->g_lag_in_frames);
1000 }
1001
1002 DCHECK(size.width());
1003 DCHECK(size.height());
1004 codec_config->g_w = size.width();
1005 codec_config->g_h = size.height();
1006 codec_config->g_pass = VPX_RC_ONE_PASS;
1007
1008 // Timebase is the smallest interval used by the stream, can be set to the
1009 // frame rate or to e.g. microseconds.
1010 codec_config->g_timebase.num = 1;
1011 codec_config->g_timebase.den = base::Time::kMicrosecondsPerSecond;
1012
1013 // Let the encoder decide where to place the Keyframes, between min and max.
1014 // In VPX_KF_AUTO mode libvpx will sometimes emit keyframes regardless of min/
1015 // max distance out of necessity.
1016 // Note that due to http://crbug.com/440223, it might be necessary to force a
1017 // key frame after 10,000frames since decoding fails after 30,000 non-key
1018 // frames.
1019 // Forcing a keyframe in regular intervals also allows seeking in the
1020 // resulting recording with decent performance.
1021 codec_config->kf_mode = VPX_KF_AUTO;
1022 codec_config->kf_min_dist = 0;
1023 codec_config->kf_max_dist = 100;
1024
1025 codec_config->g_threads = GetNumberOfThreadsForEncoding();
1026
1027 // Number of frames to consume before producing output.
1028 codec_config->g_lag_in_frames = 0;
1029
1030 encoder->reset(new vpx_codec_ctx_t);
1031 const vpx_codec_err_t ret = vpx_codec_enc_init(
1032 encoder->get(), codec_interface, codec_config, 0 /* flags */);
1033 DCHECK_EQ(VPX_CODEC_OK, ret);
1034
1035 if (use_vp9_) {
1036 // Values of VP8E_SET_CPUUSED greater than 0 will increase encoder speed at
1037 // the expense of quality up to a maximum value of 8 for VP9, by tuning the
1038 // target time spent encoding the frame. Go from 8 to 5 (values for real
1039 // time encoding) depending on the amount of cores available in the system.
1040 const int kCpuUsed =
1041 std::max(5, 8 - base::SysInfo::NumberOfProcessors() / 2);
1042 result = vpx_codec_control(encoder->get(), VP8E_SET_CPUUSED, kCpuUsed);
1043 DLOG_IF(WARNING, VPX_CODEC_OK != result) << "VP8E_SET_CPUUSED failed";
1044 }
1045 }
1046
1047 bool VpxEncoder::IsInitialized(const vpx_codec_enc_cfg_t& codec_config) const {
1048 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
1049 return codec_config.g_timebase.den != 0;
1050 }
1051
1052 base::TimeDelta VpxEncoder::EstimateFrameDuration(
1053 const scoped_refptr<VideoFrame>& frame) {
1054 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
1055
1056 using base::TimeDelta;
1057 TimeDelta predicted_frame_duration;
1058 if (!frame->metadata()->GetTimeDelta(VideoFrameMetadata::FRAME_DURATION,
1059 &predicted_frame_duration) ||
1060 predicted_frame_duration <= TimeDelta()) {
1061 // The source of the video frame did not provide the frame duration. Use
1062 // the actual amount of time between the current and previous frame as a
1063 // prediction for the next frame's duration.
1064 // TODO(mcasas): This duration estimation could lead to artifacts if the
1065 // cadence of the received stream is compromised (e.g. camera freeze, pause,
1066 // remote packet loss). Investigate using GetFrameRate() in this case.
1067 predicted_frame_duration = frame->timestamp() - last_frame_timestamp_;
1068 }
1069 last_frame_timestamp_ = frame->timestamp();
1070 // Make sure |predicted_frame_duration| is in a safe range of values.
1071 const TimeDelta kMaxFrameDuration = TimeDelta::FromSecondsD(1.0 / 8);
1072 const TimeDelta kMinFrameDuration = TimeDelta::FromMilliseconds(1);
1073 return std::min(kMaxFrameDuration, std::max(predicted_frame_duration,
1074 kMinFrameDuration));
1075 }
1076
1077 #if BUILDFLAG(RTC_USE_H264)
1078
1079 // static
1080 void H264Encoder::ShutdownEncoder(std::unique_ptr<base::Thread> encoding_thread,
1081 ScopedISVCEncoderPtr encoder) {
1082 DCHECK(encoding_thread->IsRunning());
1083 encoding_thread->Stop();
1084 // Both |encoding_thread| and |encoder| will be destroyed at end-of-scope.
1085 }
1086
1087 H264Encoder::H264Encoder(
1088 const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
1089 int32_t bits_per_second)
1090 : Encoder(on_encoded_video_callback, bits_per_second) {
1091 DCHECK(encoding_thread_->IsRunning());
1092 }
1093
1094 H264Encoder::~H264Encoder() {
1095 main_task_runner_->PostTask(FROM_HERE,
1096 base::Bind(&H264Encoder::ShutdownEncoder,
1097 base::Passed(&encoding_thread_),
1098 base::Passed(&openh264_encoder_)));
1099 }
1100
1101 void H264Encoder::EncodeOnEncodingTaskRunner(
1102 scoped_refptr<VideoFrame> frame,
1103 base::TimeTicks capture_timestamp) {
1104 TRACE_EVENT0("video", "H264Encoder::EncodeOnEncodingTaskRunner");
1105 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
1106
1107 const gfx::Size frame_size = frame->visible_rect().size();
1108 if (!openh264_encoder_ || configured_size_ != frame_size) {
1109 ConfigureEncoderOnEncodingTaskRunner(frame_size);
1110 first_frame_timestamp_ = capture_timestamp;
1111 }
1112
1113 SSourcePicture picture = {};
1114 picture.iPicWidth = frame_size.width();
1115 picture.iPicHeight = frame_size.height();
1116 picture.iColorFormat = EVideoFormatType::videoFormatI420;
1117 picture.uiTimeStamp =
1118 (capture_timestamp - first_frame_timestamp_).InMilliseconds();
1119 picture.iStride[0] = frame->stride(VideoFrame::kYPlane);
1120 picture.iStride[1] = frame->stride(VideoFrame::kUPlane);
1121 picture.iStride[2] = frame->stride(VideoFrame::kVPlane);
1122 picture.pData[0] = frame->visible_data(VideoFrame::kYPlane);
1123 picture.pData[1] = frame->visible_data(VideoFrame::kUPlane);
1124 picture.pData[2] = frame->visible_data(VideoFrame::kVPlane);
1125
1126 SFrameBSInfo info = {};
1127 if (openh264_encoder_->EncodeFrame(&picture, &info) != cmResultSuccess) {
1128 NOTREACHED() << "OpenH264 encoding failed";
1129 return;
1130 }
1131 const media::WebmMuxer::VideoParameters video_params(frame);
1132 frame = nullptr;
1133
1134 std::unique_ptr<std::string> data(new std::string);
1135 const uint8_t kNALStartCode[4] = {0, 0, 0, 1};
1136 for (int layer = 0; layer < info.iLayerNum; ++layer) {
1137 const SLayerBSInfo& layerInfo = info.sLayerInfo[layer];
1138 // Iterate NAL units making up this layer, noting fragments.
1139 size_t layer_len = 0;
1140 for (int nal = 0; nal < layerInfo.iNalCount; ++nal) {
1141 // The following DCHECKs make sure that the header of each NAL unit is OK.
1142 DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4);
1143 DCHECK_EQ(kNALStartCode[0], layerInfo.pBsBuf[layer_len+0]);
1144 DCHECK_EQ(kNALStartCode[1], layerInfo.pBsBuf[layer_len+1]);
1145 DCHECK_EQ(kNALStartCode[2], layerInfo.pBsBuf[layer_len+2]);
1146 DCHECK_EQ(kNALStartCode[3], layerInfo.pBsBuf[layer_len+3]);
1147
1148 layer_len += layerInfo.pNalLengthInByte[nal];
1149 }
1150 // Copy the entire layer's data (including NAL start codes).
1151 data->append(reinterpret_cast<char*>(layerInfo.pBsBuf), layer_len);
1152 }
1153
1154 const bool is_key_frame = info.eFrameType == videoFrameTypeIDR;
1155 origin_task_runner_->PostTask(
1156 FROM_HERE, base::Bind(OnFrameEncodeCompleted, on_encoded_video_callback_,
1157 video_params, base::Passed(&data), nullptr,
1158 capture_timestamp, is_key_frame));
1159 }
1160
1161 void H264Encoder::ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) {
1162 DCHECK(encoding_task_runner_->BelongsToCurrentThread());
1163 ISVCEncoder* temp_encoder = nullptr;
1164 if (WelsCreateSVCEncoder(&temp_encoder) != 0) {
1165 NOTREACHED() << "Failed to create OpenH264 encoder";
1166 return;
1167 }
1168 openh264_encoder_.reset(temp_encoder);
1169 configured_size_ = size;
1170
1171 #if DCHECK_IS_ON()
1172 int trace_level = WELS_LOG_INFO;
1173 openh264_encoder_->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
1174 #endif
1175
1176 SEncParamExt init_params;
1177 openh264_encoder_->GetDefaultParams(&init_params);
1178 init_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
1179
1180 DCHECK_EQ(AUTO_REF_PIC_COUNT, init_params.iNumRefFrame);
1181 DCHECK(!init_params.bSimulcastAVC);
1182
1183 init_params.uiIntraPeriod = 100; // Same as for VpxEncoder.
1184 init_params.iPicWidth = size.width();
1185 init_params.iPicHeight = size.height();
1186
1187 DCHECK_EQ(RC_QUALITY_MODE, init_params.iRCMode);
1188 DCHECK_EQ(0, init_params.iPaddingFlag);
1189 DCHECK_EQ(UNSPECIFIED_BIT_RATE, init_params.iTargetBitrate);
1190 DCHECK_EQ(UNSPECIFIED_BIT_RATE, init_params.iMaxBitrate);
1191 if (bits_per_second_ > 0) {
1192 init_params.iRCMode = RC_BITRATE_MODE;
1193 init_params.iTargetBitrate = bits_per_second_;
1194 } else {
1195 init_params.iRCMode = RC_OFF_MODE;
1196 }
1197
1198 // Threading model: Set to 1 due to https://crbug.com/583348.
1199 init_params.iMultipleThreadIdc = 1;
1200
1201 // TODO(mcasas): consider reducing complexity if there are few CPUs available.
1202 init_params.iComplexityMode = MEDIUM_COMPLEXITY;
1203 DCHECK(!init_params.bEnableDenoise);
1204 DCHECK(init_params.bEnableFrameSkip);
1205
1206 // The base spatial layer 0 is the only one we use.
1207 DCHECK_EQ(1, init_params.iSpatialLayerNum);
1208 init_params.sSpatialLayers[0].iVideoWidth = init_params.iPicWidth;
1209 init_params.sSpatialLayers[0].iVideoHeight = init_params.iPicHeight;
1210 init_params.sSpatialLayers[0].iSpatialBitrate = init_params.iTargetBitrate;
1211
1212 // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto design
1213 // it with cpu core number.
1214 // TODO(sprang): Set to 0 when we understand why the rate controller borks
1215 // when uiSliceNum > 1. See https://github.com/cisco/openh264/issues/2591
1216 init_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
1217 init_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
1218 SM_FIXEDSLCNUM_SLICE;
1219
1220 if (openh264_encoder_->InitializeExt(&init_params) != cmResultSuccess) {
1221 NOTREACHED() << "Failed to initialize OpenH264 encoder";
1222 return;
1223 }
1224
1225 int pixel_format = EVideoFormatType::videoFormatI420;
1226 openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT, &pixel_format);
1227 }
1228 #endif //#if BUILDFLAG(RTC_USE_H264)
1229
1230 } // anonymous namespace
1231
1232 // static
1233 VideoTrackRecorder::CodecId VideoTrackRecorder::GetPreferredCodecId() { 308 VideoTrackRecorder::CodecId VideoTrackRecorder::GetPreferredCodecId() {
1234 return GetCodecEnumerator()->GetPreferredCodecId(); 309 return GetCodecEnumerator()->GetPreferredCodecId();
1235 } 310 }
1236 311
1237 VideoTrackRecorder::VideoTrackRecorder( 312 VideoTrackRecorder::VideoTrackRecorder(
1238 CodecId codec, 313 CodecId codec,
1239 const blink::WebMediaStreamTrack& track, 314 const blink::WebMediaStreamTrack& track,
1240 const OnEncodedVideoCB& on_encoded_video_callback, 315 const OnEncodedVideoCB& on_encoded_video_callback,
1241 int32_t bits_per_second) 316 int32_t bits_per_second)
1242 : track_(track), 317 : track_(track),
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after
1359 // thread. 434 // thread.
1360 MediaStreamVideoSink::DisconnectFromTrack(); 435 MediaStreamVideoSink::DisconnectFromTrack();
1361 encoder_ = nullptr; 436 encoder_ = nullptr;
1362 MediaStreamVideoSink::ConnectToTrack( 437 MediaStreamVideoSink::ConnectToTrack(
1363 track_, 438 track_,
1364 media::BindToCurrentLoop(base::Bind(initialize_encoder_callback_, 439 media::BindToCurrentLoop(base::Bind(initialize_encoder_callback_,
1365 false /*allow_vea_encoder*/)), 440 false /*allow_vea_encoder*/)),
1366 false); 441 false);
1367 } 442 }
1368 443
1369 bool VideoTrackRecorder::CanEncodeAlphaChannelForTesting() {
1370 DCHECK(encoder_);
1371 return encoder_->CanEncodeAlphaChannel();
1372 }
1373
1374 } // namespace content 444 } // namespace content
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698