Support alpha channel recording for VPX in MediaRecorder

media/muxers/webm_muxer.cc

Index: media/muxers/webm_muxer.cc
diff --git a/media/muxers/webm_muxer.cc b/media/muxers/webm_muxer.cc
index 02a4ca9fb2bcbe51b872c5b9f8fc3046c353b264..1ab9cc018cda0b40ed9b745c7f8bd75f0cb8e065 100644
--- a/media/muxers/webm_muxer.cc
+++ b/media/muxers/webm_muxer.cc
@@ -134,6 +134,7 @@ WebmMuxer::~WebmMuxer() {
 
 bool WebmMuxer::OnEncodedVideo(const VideoParameters& params,
                                std::unique_ptr<std::string> encoded_data,
+                               std::unique_ptr<std::string> encoded_alpha_data,
                                base::TimeTicks timestamp,
                                bool is_key_frame) {
   DVLOG(1) << __func__ << " - " << encoded_data->size() << "B";
@@ -154,15 +155,17 @@ bool WebmMuxer::OnEncodedVideo(const VideoParameters& params,
       encoded_frames_queue_.clear();
 
     encoded_frames_queue_.push_back(base::MakeUnique<EncodedVideoFrame>(
-        std::move(encoded_data), timestamp, is_key_frame));
+        std::move(encoded_data), std::move(encoded_alpha_data), timestamp,
+        is_key_frame));
     return true;
   }
 
   // Any saved encoded video frames must have been dumped in OnEncodedAudio();
   DCHECK(encoded_frames_queue_.empty());
 
-  return AddFrame(std::move(encoded_data), video_track_index_,
-                  timestamp - first_frame_timestamp_video_, is_key_frame);
+  return AddFrame(std::move(encoded_data), std::move(encoded_alpha_data),
+                  video_track_index_, timestamp - first_frame_timestamp_video_,
+                  is_key_frame);
 }
 
 bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
@@ -188,6 +191,10 @@ bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
   while (!encoded_frames_queue_.empty()) {
     const bool res = AddFrame(
         base::MakeUnique<std::string>(*encoded_frames_queue_.front()->data),
+        encoded_frames_queue_.front()->alpha_data
+            ? base::MakeUnique<std::string>(
+                  *encoded_frames_queue_.front()->alpha_data)
+            : nullptr,
         video_track_index_,
         encoded_frames_queue_.front()->timestamp - first_frame_timestamp_video_,
         encoded_frames_queue_.front()->is_keyframe);
@@ -195,8 +202,7 @@ bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
       return false;
     encoded_frames_queue_.pop_front();
   }
-
-  return AddFrame(std::move(encoded_data), audio_track_index_,
+  return AddFrame(std::move(encoded_data), nullptr, audio_track_index_,
                   timestamp - first_frame_timestamp_audio_,
                   true /* is_key_frame -- always true for audio */);
 }
@@ -240,6 +246,9 @@ void WebmMuxer::AddVideoTrack(const gfx::Size& frame_size, double frame_rate) {
   DCHECK_EQ(0ull, video_track->crop_bottom());
   DCHECK_EQ(0.0f, video_track->frame_rate());
 
+  video_track->SetAlphaMode(mkvmuxer::VideoTrack::kAlpha);

[mcasas, 2017/03/08 22:38:48]

Do we have to do it always, regardless of
alpha being produced in the encoder or not?

[emircan, 2017/03/09 01:45:41]

According to my offline discussion with vignesh@, yes. We need to set this
before encode actually starts and we do not know if alpha is going to be
included or not.

+  video_track->set_max_block_additional_id(1);

[mcasas, 2017/03/08 22:38:48]

We need a comment as to why we make this additional
thing. Link e.g. to http://wiki.webmproject.org/alpha-channel
and say that Alpha would go as a BlockAdditional or so.

Also, if you are (over)writing it just to make sure
of the value, DCHECK() it instead as done in e.g. l254.

[emircan, 2017/03/09 01:45:41]

Done.

+
   // Segment's timestamps should be in milliseconds, DCHECK it. See
   // http://www.webmproject.org/docs/container/#muxer-guidelines
   DCHECK_EQ(1000000ull, segment_.GetSegmentInfo()->timecode_scale());
@@ -308,6 +317,7 @@ void WebmMuxer::ElementStartNotify(mkvmuxer::uint64 element_id,
 }
 
 bool WebmMuxer::AddFrame(std::unique_ptr<std::string> encoded_data,
+                         std::unique_ptr<std::string> encoded_alpha_data,
                          uint8_t track_index,
                          base::TimeDelta timestamp,
                          bool is_key_frame) {
@@ -325,9 +335,21 @@ bool WebmMuxer::AddFrame(std::unique_ptr<std::string> encoded_data,
   }
 
   DCHECK(encoded_data->data());
-  return segment_.AddFrame(
+  if (!encoded_alpha_data || encoded_alpha_data->empty()) {

[vignesh, 2017/03/08 17:26:14]

Just checking: Have you verified if this combination of Blocks and SimpleBlocks
plays back ok in Chrome (please check both MSE and <video> tag as they use
different demuxer implementations).

[emircan, 2017/03/08 23:53:58]

It tested a alpha-opaque-alpha canvas recording on <video> tag and it works
fine. What do you refer to by MSE?

[vignesh, 2017/03/09 07:13:46]

MSE is Media Source Extensions [1]. It's how most javascript players playback
video.

Can you please share a sample file created this way with me? Just curious. We
can take this offline.

[1]
https://developers.google.com/web/fundamentals/getting-started/primers/media-...

[emircan, 2017/03/09 19:18:53]

Sure. The way we are playing back Blob in the demo page seems to be using MSE
and it works fine. I am sending a hybrid opaque-transparent webm to you via
email.

+    return segment_.AddFrame(
+        reinterpret_cast<const uint8_t*>(encoded_data->data()),
+        encoded_data->size(), track_index,
+        most_recent_timestamp_.InMicroseconds() *
+            base::Time::kNanosecondsPerMicrosecond,
+        is_key_frame);
+  }
+
+  DCHECK(encoded_alpha_data->data());
+  return segment_.AddFrameWithAdditional(
       reinterpret_cast<const uint8_t*>(encoded_data->data()),
-      encoded_data->size(), track_index,
+      encoded_data->size(),
+      reinterpret_cast<const uint8_t*>(encoded_alpha_data->data()),
+      encoded_alpha_data->size(), 1, track_index,

[mcasas, 2017/03/08 22:38:48]

What's this "1"? It's
 /* add_id */

[emircan, 2017/03/09 01:45:40]

Added the comment. I just followed this from the webm sample.

       most_recent_timestamp_.InMicroseconds() *
           base::Time::kNanosecondsPerMicrosecond,
       is_key_frame);
@@ -335,9 +357,13 @@ bool WebmMuxer::AddFrame(std::unique_ptr<std::string> encoded_data,
 
 WebmMuxer::EncodedVideoFrame::EncodedVideoFrame(
     std::unique_ptr<std::string> data,
+    std::unique_ptr<std::string> alpha_data,
     base::TimeTicks timestamp,
     bool is_keyframe)
-    : data(std::move(data)), timestamp(timestamp), is_keyframe(is_keyframe) {}
+    : data(std::move(data)),
+      alpha_data(std::move(alpha_data)),
+      timestamp(timestamp),
+      is_keyframe(is_keyframe) {}
 
 WebmMuxer::EncodedVideoFrame::~EncodedVideoFrame() {}