Handle scaling frames in RTCVideoEncoder

content/renderer/media/gpu/rtc_video_encoder.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/gpu/rtc_video_encoder.h"
 
 #include <string.h>
 
 #include "base/bind.h"
 #include "base/location.h"
@@ skipping 567 matching lines @@
     frame = media::VideoFrame::WrapExternalSharedMemory(
         media::PIXEL_FORMAT_I420, input_frame_coded_size_,
         gfx::Rect(input_visible_size_), input_visible_size_,
         reinterpret_cast<uint8_t*>(input_buffer->memory()),
         input_buffer->mapped_size(), input_buffer->handle(), 0, timestamp);
     if (!frame.get()) {
       LogAndNotifyError(FROM_HERE, "failed to create frame",
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
-    // Do a strided copy of the input frame to match the input requirements for
-    // the encoder.
-    // TODO(sheu): support zero-copy from WebRTC.  http://crbug.com/269312
-    if (libyuv::I420Copy(next_frame->video_frame_buffer()->DataY(),
-                         next_frame->video_frame_buffer()->StrideY(),
-                         next_frame->video_frame_buffer()->DataU(),
-                         next_frame->video_frame_buffer()->StrideU(),
-                         next_frame->video_frame_buffer()->DataV(),
-                         next_frame->video_frame_buffer()->StrideV(),
-                         frame->data(media::VideoFrame::kYPlane),
-                         frame->stride(media::VideoFrame::kYPlane),
-                         frame->data(media::VideoFrame::kUPlane),
-                         frame->stride(media::VideoFrame::kUPlane),
-                         frame->data(media::VideoFrame::kVPlane),
-                         frame->stride(media::VideoFrame::kVPlane),
-                         next_frame->width(), next_frame->height())) {
+
+    // Do a strided copy and scale (if necessary) the input frame to match
+    // the input requirements for the encoder.
+    // TODO(sheu): support zero-copy from WebRTC. http://crbug.com/269312
+    if (libyuv::I420Scale(next_frame->video_frame_buffer()->DataY(),
+                          next_frame->video_frame_buffer()->StrideY(),
+                          next_frame->video_frame_buffer()->DataU(),
+                          next_frame->video_frame_buffer()->StrideU(),
+                          next_frame->video_frame_buffer()->DataV(),
+                          next_frame->video_frame_buffer()->StrideV(),
+                          next_frame->width(), next_frame->height(),
+                          frame->visible_data(media::VideoFrame::kYPlane),
+                          frame->stride(media::VideoFrame::kYPlane),
+                          frame->visible_data(media::VideoFrame::kUPlane),
+                          frame->stride(media::VideoFrame::kUPlane),
+                          frame->visible_data(media::VideoFrame::kVPlane),
+                          frame->stride(media::VideoFrame::kVPlane),
+                          frame->visible_rect().width(),
+                          frame->visible_rect().height(),
+                          libyuv::kFilterBilinear)) {

[Pawel Osciak, 2016/07/28 01:48:08]

Do we have an idea of the performance impact of bilinear filtering? I'm
wondering if we could perhaps roughly measure it with the unittest by doing a
copy and comparing to scaling?

[tommi (sloooow) - chröme, 2016/07/28 07:40:40]

+1

[magjed_chromium, 2016/07/28 10:27:19]

I think we should use libyuv::kFilterBox instead of libyuv::kFilterBilinear, to
avoid aliasing when downsampling. libyuv::kFilterBilinear means bilinear point
sampling and is faster than proper box scaling, but the performance difference
is negligible for optimized 1/2 and 1/4 scaling. Scaling from 1280x720 to
176x144 is not optimized though. The optimal solution in terms of quality and
performance is to do an image pyramid with box scaling 1280x720 -> 640x360 ->
320x180 -> 176x144, instead of 1280x720 -> 640x360, 1280x720 -> 320x180, and
1280x720 -> 176x144, but I'm not sure how to implement that since the
RTCVideoEncoders are unaware of each other.

SimulcastEncoderAdapter uses libyuv::kFilterBilinear, so using it here as well
would not be a quality regression.

I did a quick benchmark of libyuv scaling (each operation is repeated 1000
times, so cache hit is optimal, and these numbers might be too low compared to
real use). 
Using Neon (on a Nexus 5):
                   Bilinear Box
1280x720->176x144: 0.21ms, 4.51ms
1280x720->320x180: 0.48ms, 0.40ms
1280x720->640x360: 0.35ms, 0.33ms 
1280x720 copy:     1.14ms

x86 (AVX) (on my workstation):
                   Bilinear Box
1280x720->176x144: 0.07ms, 1.04ms
1280x720->320x180: 0.14ms, 0.06ms
1280x720->640x360: 0.09ms, 0.09ms
1280x720 copy:     0.13ms

So 1280x720->176x144 with box scaling is ridiculously slow, it's actually faster
to do 1280x720->640x360->320x180->176x144. I will put it on my todo list to fix
that in libyuv.

[emircan, 2016/07/28 19:35:45]

SimulcastEncoderAdapter uses libyuv::kFilterBilinear, so there wouldn't be a
quality or performance difference if there is scaling in comparison to before.

If input and output sizes are the same, it falls back to just copying the
frame[0]. It will be the same codepath as libyuv::I420Copy before[1] hence the
same performance. I tried to note that in the CL description as well.

[0]
https://cs.chromium.org/chromium/src/third_party/libyuv/source/scale.cc?rcl=0...
[1]
https://cs.chromium.org/chromium/src/third_party/libyuv/source/convert.cc?rcl...

[emircan, 2016/07/28 19:35:45]

Thanks for the metrics. Would you recommend changing to libyuv::kFilterBox right
now despite the performance? Or we can add a TODO here to followup after the
improvements libyuv::kFilterBox rolls in?

[magjed_chromium, 2016/07/30 08:30:17]

I think it's ok to use kFilterBilinear for now. You can put the TODO on me,
something like:
TODO(magjed): Downscale with kFilterBox in an image pyramid instead.

       LogAndNotifyError(FROM_HERE, "Failed to copy buffer",
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
   }
   frame->AddDestructionObserver(media::BindToCurrentLoop(
       base::Bind(&RTCVideoEncoder::Impl::EncodeFrameFinished, this, index)));
   video_encoder_->Encode(frame, next_frame_keyframe);
   input_buffers_free_.pop_back();
   SignalAsyncWaiter(WEBRTC_VIDEO_CODEC_OK);
@@ skipping 260 matching lines @@
   UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoEncoderInitEncodeSuccess",
                         init_retval == WEBRTC_VIDEO_CODEC_OK);
   if (init_retval == WEBRTC_VIDEO_CODEC_OK) {
     UMA_HISTOGRAM_ENUMERATION("Media.RTCVideoEncoderProfile",
                               profile,
                               media::VIDEO_CODEC_PROFILE_MAX + 1);
   }
 }
 
 }  // namespace content