VideoTrackRecorder: add support for texture backed ARGB VideoFrames

content/renderer/media/video_track_recorder.cc

Index: content/renderer/media/video_track_recorder.cc
diff --git a/content/renderer/media/video_track_recorder.cc b/content/renderer/media/video_track_recorder.cc
index e5d477a18be2037f4f23a7461e7a8a65cdb9f370..4baceb4dd27ea91008efe6f0d9045336de8a620f 100644
--- a/content/renderer/media/video_track_recorder.cc
+++ b/content/renderer/media/video_track_recorder.cc
@@ -15,12 +15,18 @@
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
+#include "content/common/gpu/client/context_provider_command_buffer.h"
 #include "content/renderer/media/renderer_gpu_video_accelerator_factories.h"
 #include "content/renderer/render_thread_impl.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
-#include "third_party/libyuv/include/libyuv/convert.h"
+#include "media/filters/context_3d.h"
+#include "media/renderers/skcanvas_video_renderer.h"
+#include "skia/ext/platform_canvas.h"
+#include "third_party/libyuv/include/libyuv.h"
+#include "third_party/skia/include/core/SkCanvas.h"
+#include "third_party/skia/include/core/SkSurface.h"
 #include "ui/gfx/geometry/size.h"
 
 #if BUILDFLAG(RTC_USE_H264)
@@ -105,8 +111,8 @@ media::VideoCodecProfile CodecIdToVEAProfile(
 // ref-counted to allow the MediaStreamVideoTrack to hold a reference to it (via
 // the callback that MediaStreamVideoSink passes along) and to jump back and
 // forth to an internal encoder thread. Moreover, this class:
-// - is created and destroyed on its parent's thread (usually the main Render
-// thread), |main_task_runner_|.
+// - is created on its parent's thread (usually the main Render thread),
+// that is, |main_task_runner_|.
 // - receives VideoFrames on |origin_task_runner_| and runs OnEncodedVideoCB on
 // that thread as well. This task runner is cached on first frame arrival, and
 // is supposed to be the render IO thread (but this is not enforced);
@@ -136,15 +142,21 @@ class VideoTrackRecorder::Encoder : public base::RefCountedThreadSafe<Encoder> {
   // Start encoding |frame|, returning via |on_encoded_video_callback_|. This
   // call will also trigger a ConfigureEncoderOnEncodingTaskRunner() upon first
   // frame arrival or parameter change, and an EncodeOnEncodingTaskRunner() to
-  // actually encode the frame.
+  // actually encode the frame. If the |frame|'s data is not directly available
+  // (e.g. it's a texture) then RetrieveFrameOnMainThread() is called, and if
+  // even that fails, black frames are sent instead.
   void StartFrameEncode(const scoped_refptr<VideoFrame>& frame,
                         base::TimeTicks capture_timestamp);
+  void RetrieveFrameOnMainThread(const scoped_refptr<VideoFrame>& video_frame,
+                                 base::TimeTicks capture_timestamp);
 
   void SetPaused(bool paused);
 
  protected:
   friend class base::RefCountedThreadSafe<Encoder>;
-  virtual ~Encoder() {}
+  virtual ~Encoder() {
+    main_task_runner_->DeleteSoon(FROM_HERE, video_renderer_.release());
+  }
 
   virtual void EncodeOnEncodingTaskRunner(
       const scoped_refptr<VideoFrame>& frame,
@@ -172,6 +184,11 @@ class VideoTrackRecorder::Encoder : public base::RefCountedThreadSafe<Encoder> {
   // Target bitrate for video encoding. If 0, a standard bitrate is used.
   const int32_t bits_per_second_;
 
+  // Used to retrieve incoming opaque VideoFrames (i.e. VideoFrames backed by
+  // textures). Created on-demand on |main_task_runner_|.
+  std::unique_ptr<media::SkCanvasVideoRenderer> video_renderer_;
+  sk_sp<SkSurface> surface_;
+
   DISALLOW_COPY_AND_ASSIGN(Encoder);
 };
 
@@ -187,10 +204,19 @@ void VideoTrackRecorder::Encoder::StartFrameEncode(
 
   if (!(video_frame->format() == media::PIXEL_FORMAT_I420 ||
         video_frame->format() == media::PIXEL_FORMAT_YV12 ||
+        video_frame->format() == media::PIXEL_FORMAT_ARGB ||
         video_frame->format() == media::PIXEL_FORMAT_YV12A)) {
-    NOTREACHED();
+    NOTREACHED() << media::VideoPixelFormatToString(video_frame->format());
     return;
   }
+
+  if (video_frame->HasTextures()) {
+    main_task_runner_->PostTask(
+        FROM_HERE, base::Bind(&Encoder::RetrieveFrameOnMainThread, this,
+                              video_frame, capture_timestamp));
+    return;
+  }
+
   scoped_refptr<media::VideoFrame> frame = video_frame;
   // Drop alpha channel since we do not support it yet.
   if (frame->format() == media::PIXEL_FORMAT_YV12A)
@@ -201,6 +227,80 @@ void VideoTrackRecorder::Encoder::StartFrameEncode(
                             capture_timestamp));
 }
 
+void VideoTrackRecorder::Encoder::RetrieveFrameOnMainThread(
+    const scoped_refptr<VideoFrame>& video_frame,
+    base::TimeTicks capture_timestamp) {
+  DCHECK(main_task_runner_->BelongsToCurrentThread());
+
+  scoped_refptr<media::VideoFrame> frame;
+
+  // |context_provider| is null if the GPU process has crashed or isn't there.
+  ContextProviderCommandBuffer* const context_provider =
+      RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
+  if (!context_provider) {
+    // Send black frames (yuv = {0, 127, 127}).
+    frame = media::VideoFrame::CreateColorFrame(
+        video_frame->coded_size(), 0u, 0x80, 0x80, video_frame->timestamp());

[emircan, 2016/07/15 17:47:34]

Replace coded_size() with visible_rect() here. CreateColorFrame() takes only one
param, so keeping visible_rect() the same would be a better option.

[mcasas, 2016/07/15 18:03:09]

Done.

+  } else {
+    // Accelerated decoders produce ARGB/ABGR texture-backed frames (see
+    // https://crbug.com/585242), fetch them using a SkCanvasVideoRenderer.
+    DCHECK(video_frame->HasTextures());
+    DCHECK_EQ(media::PIXEL_FORMAT_ARGB, video_frame->format());
+
+    frame = media::VideoFrame::CreateFrame(
+        media::PIXEL_FORMAT_I420, video_frame->coded_size(),
+        video_frame->visible_rect(), video_frame->natural_size(),
+        video_frame->timestamp());
+
+    const SkImageInfo info = SkImageInfo::MakeN32(
+        frame->visible_rect().width(), frame->visible_rect().height(),
+        kOpaque_SkAlphaType);
+
+    // Create |surface_| if it doesn't exist or incoming resolution has changed.
+    if (!surface_ || surface_->width() != info.width() ||
+        surface_->height() != info.height()) {
+      surface_ = SkSurface::MakeRaster(info);
+    }
+    if (!video_renderer_)
+      video_renderer_.reset(new media::SkCanvasVideoRenderer);
+
+    DCHECK(context_provider->ContextGL());
+    video_renderer_->Copy(video_frame.get(), surface_->getCanvas(),
+                          media::Context3D(context_provider->ContextGL(),
+                                           context_provider->GrContext()));
+
+    SkPixmap pixmap;
+    if (!skia::GetWritablePixels(surface_->getCanvas(), &pixmap)) {
+      DLOG(ERROR) << "Error trying to map SkSurface's pixels";
+      return;
+    }
+
+    const uint32 source_pixel_format =
+        (kN32_SkColorType == kRGBA_8888_SkColorType) ? libyuv::FOURCC_ABGR
+                                                     : libyuv::FOURCC_ARGB;
+    if (libyuv::ConvertToI420(static_cast<uint8*>(pixmap.writable_addr()),
+                              pixmap.getSafeSize(),
+                              frame->visible_data(media::VideoFrame::kYPlane),
+                              frame->stride(media::VideoFrame::kYPlane),
+                              frame->visible_data(media::VideoFrame::kUPlane),
+                              frame->stride(media::VideoFrame::kUPlane),
+                              frame->visible_data(media::VideoFrame::kVPlane),
+                              frame->stride(media::VideoFrame::kVPlane),
+                              0 /* crop_x */, 0 /* crop_y */,
+                              pixmap.width(), pixmap.height(),
+                              frame->coded_size().width(),

[emircan, 2016/07/15 17:47:34]

Replace coded_size() with visible_rect() here. You allocated SkPixmap with
visible_rect on l.256 and copying into visible_data(). 

[mcasas, 2016/07/15 18:03:09]

Done.

+                              frame->coded_size().height(),
+                              libyuv::kRotate0, source_pixel_format) != 0) {

[emircan, 2016/07/15 17:47:34]

Consider the case for rotation. We have seen that remote Android peer with H264
result in textures with rotation [0]. If that CL gets committed earlier, you can
use the frame metadata.

[0] https://codereview.chromium.org/2150203002/

[mcasas, 2016/07/15 18:03:09]

Added TODO, will revisit.

+      DLOG(ERROR) << "Error converting frame to I420";
+      return;
+    }
+  }
+
+  encoding_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&Encoder::EncodeOnEncodingTaskRunner, this, frame,
+                            capture_timestamp));
+}
+
 void VideoTrackRecorder::Encoder::SetPaused(bool paused) {
   if (!encoding_task_runner_->BelongsToCurrentThread()) {
     encoding_task_runner_->PostTask(