Support alpha channel recording for VPX in MediaRecorder

media/muxers/webm_muxer.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/muxers/webm_muxer.h"
 
 #include <algorithm>
 
 #include "base/bind.h"
 #include "base/memory/ptr_util.h"
@@ skipping 116 matching lines @@
 
 WebmMuxer::~WebmMuxer() {
   // No need to segment_.Finalize() since is not Seekable(), i.e. a live
   // stream, but is a good practice.
   DCHECK(thread_checker_.CalledOnValidThread());
   segment_.Finalize();
 }
 
 bool WebmMuxer::OnEncodedVideo(const VideoParameters& params,
                                std::unique_ptr<std::string> encoded_data,
+                               std::unique_ptr<std::string> encoded_alpha_data,
                                base::TimeTicks timestamp,
                                bool is_key_frame) {
   DVLOG(1) << __func__ << " - " << encoded_data->size() << "B";
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (!video_track_index_) {
     // |track_index_|, cannot be zero (!), initialize WebmMuxer in that case.
     // http://www.matroska.org/technical/specs/index.html#Tracks
     AddVideoTrack(params.visible_rect_size, GetFrameRate(params));
     if (first_frame_timestamp_video_.is_null())
       first_frame_timestamp_video_ = timestamp;
   }
 
   // TODO(ajose): Support multiple tracks: http://crbug.com/528523
   if (has_audio_ && !audio_track_index_) {
     DVLOG(1) << __func__ << ": delaying until audio track ready.";
     if (is_key_frame)  // Upon Key frame reception, empty the encoded queue.
       encoded_frames_queue_.clear();
 
     encoded_frames_queue_.push_back(base::MakeUnique<EncodedVideoFrame>(
-        std::move(encoded_data), timestamp, is_key_frame));
+        std::move(encoded_data), std::move(encoded_alpha_data), timestamp,
+        is_key_frame));
     return true;
   }
 
   // Any saved encoded video frames must have been dumped in OnEncodedAudio();
   DCHECK(encoded_frames_queue_.empty());
 
-  return AddFrame(std::move(encoded_data), video_track_index_,
-                  timestamp - first_frame_timestamp_video_, is_key_frame);
+  return AddFrame(std::move(encoded_data), std::move(encoded_alpha_data),
+                  video_track_index_, timestamp - first_frame_timestamp_video_,
+                  is_key_frame);
 }
 
 bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
                                std::unique_ptr<std::string> encoded_data,
                                base::TimeTicks timestamp) {
   DVLOG(2) << __func__ << " - " << encoded_data->size() << "B";
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (!audio_track_index_) {
     AddAudioTrack(params);
     if (first_frame_timestamp_audio_.is_null())
       first_frame_timestamp_audio_ = timestamp;
   }
 
   // TODO(ajose): Don't drop audio data: http://crbug.com/547948
   // TODO(ajose): Support multiple tracks: http://crbug.com/528523
   if (has_video_ && !video_track_index_) {
     DVLOG(1) << __func__ << ": delaying until video track ready.";
     return true;
   }
 
   // Dump all saved encoded video frames if any.
   while (!encoded_frames_queue_.empty()) {
     const bool res = AddFrame(
         base::MakeUnique<std::string>(*encoded_frames_queue_.front()->data),
+        encoded_frames_queue_.front()->alpha_data
+            ? base::MakeUnique<std::string>(
+                  *encoded_frames_queue_.front()->alpha_data)
+            : nullptr,
         video_track_index_,
         encoded_frames_queue_.front()->timestamp - first_frame_timestamp_video_,
         encoded_frames_queue_.front()->is_keyframe);
     if (!res)
       return false;
     encoded_frames_queue_.pop_front();
   }
-
-  return AddFrame(std::move(encoded_data), audio_track_index_,
+  return AddFrame(std::move(encoded_data), nullptr, audio_track_index_,
                   timestamp - first_frame_timestamp_audio_,
                   true /* is_key_frame -- always true for audio */);
 }
 
 void WebmMuxer::Pause() {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
   if (!elapsed_time_in_pause_)
     elapsed_time_in_pause_.reset(new base::ElapsedTimer());
 }
@@ skipping 23 matching lines @@
       reinterpret_cast<mkvmuxer::VideoTrack*>(
           segment_.GetTrackByNumber(video_track_index_));
   DCHECK(video_track);
   video_track->set_codec_id(MkvCodeIcForMediaVideoCodecId(video_codec_));
   DCHECK_EQ(0ull, video_track->crop_right());
   DCHECK_EQ(0ull, video_track->crop_left());
   DCHECK_EQ(0ull, video_track->crop_top());
   DCHECK_EQ(0ull, video_track->crop_bottom());
   DCHECK_EQ(0.0f, video_track->frame_rate());
 
+  video_track->SetAlphaMode(mkvmuxer::VideoTrack::kAlpha);

[mcasas, 2017/03/08 22:38:48]

Do we have to do it always, regardless of
alpha being produced in the encoder or not?

[emircan, 2017/03/09 01:45:41]

According to my offline discussion with vignesh@, yes. We need to set this
before encode actually starts and we do not know if alpha is going to be
included or not.

+  video_track->set_max_block_additional_id(1);

[mcasas, 2017/03/08 22:38:48]

We need a comment as to why we make this additional
thing. Link e.g. to http://wiki.webmproject.org/alpha-channel
and say that Alpha would go as a BlockAdditional or so.

Also, if you are (over)writing it just to make sure
of the value, DCHECK() it instead as done in e.g. l254.

[emircan, 2017/03/09 01:45:41]

Done.

+
   // Segment's timestamps should be in milliseconds, DCHECK it. See
   // http://www.webmproject.org/docs/container/#muxer-guidelines
   DCHECK_EQ(1000000ull, segment_.GetSegmentInfo()->timecode_scale());
 }
 
 void WebmMuxer::AddAudioTrack(const media::AudioParameters& params) {
   DVLOG(1) << __func__ << " " << params.AsHumanReadableString();
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK_EQ(0u, audio_track_index_)
       << "WebmMuxer audio can only be initialised once.";
@@ skipping 48 matching lines @@
 }
 
 void WebmMuxer::ElementStartNotify(mkvmuxer::uint64 element_id,
                                    mkvmuxer::int64 position) {
   // This method gets pinged before items are sent to |write_data_callback_|.
   DCHECK_GE(position, position_.ValueOrDefault(0))
       << "Can't go back in a live WebM stream.";
 }
 
 bool WebmMuxer::AddFrame(std::unique_ptr<std::string> encoded_data,
+                         std::unique_ptr<std::string> encoded_alpha_data,
                          uint8_t track_index,
                          base::TimeDelta timestamp,
                          bool is_key_frame) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(!has_video_ || video_track_index_);
   DCHECK(!has_audio_ || audio_track_index_);
 
   most_recent_timestamp_ =
       std::max(most_recent_timestamp_, timestamp - total_time_in_pause_);
 
   if (force_one_libwebm_error_) {
     DVLOG(1) << "Forcing a libwebm error";
     force_one_libwebm_error_ = false;
     return false;
   }
 
   DCHECK(encoded_data->data());
-  return segment_.AddFrame(
+  if (!encoded_alpha_data || encoded_alpha_data->empty()) {

[vignesh, 2017/03/08 17:26:14]

Just checking: Have you verified if this combination of Blocks and SimpleBlocks
plays back ok in Chrome (please check both MSE and <video> tag as they use
different demuxer implementations).

[emircan, 2017/03/08 23:53:58]

It tested a alpha-opaque-alpha canvas recording on <video> tag and it works
fine. What do you refer to by MSE?

[vignesh, 2017/03/09 07:13:46]

MSE is Media Source Extensions [1]. It's how most javascript players playback
video.

Can you please share a sample file created this way with me? Just curious. We
can take this offline.

[1]
https://developers.google.com/web/fundamentals/getting-started/primers/media-...

[emircan, 2017/03/09 19:18:53]

Sure. The way we are playing back Blob in the demo page seems to be using MSE
and it works fine. I am sending a hybrid opaque-transparent webm to you via
email.

+    return segment_.AddFrame(
+        reinterpret_cast<const uint8_t*>(encoded_data->data()),
+        encoded_data->size(), track_index,
+        most_recent_timestamp_.InMicroseconds() *
+            base::Time::kNanosecondsPerMicrosecond,
+        is_key_frame);
+  }
+
+  DCHECK(encoded_alpha_data->data());
+  return segment_.AddFrameWithAdditional(
       reinterpret_cast<const uint8_t*>(encoded_data->data()),
-      encoded_data->size(), track_index,
+      encoded_data->size(),
+      reinterpret_cast<const uint8_t*>(encoded_alpha_data->data()),
+      encoded_alpha_data->size(), 1, track_index,

[mcasas, 2017/03/08 22:38:48]

What's this "1"? It's
 /* add_id */

[emircan, 2017/03/09 01:45:40]

Added the comment. I just followed this from the webm sample.

       most_recent_timestamp_.InMicroseconds() *
           base::Time::kNanosecondsPerMicrosecond,
       is_key_frame);
 }
 
 WebmMuxer::EncodedVideoFrame::EncodedVideoFrame(
     std::unique_ptr<std::string> data,
+    std::unique_ptr<std::string> alpha_data,
     base::TimeTicks timestamp,
     bool is_keyframe)
-    : data(std::move(data)), timestamp(timestamp), is_keyframe(is_keyframe) {}
+    : data(std::move(data)),
+      alpha_data(std::move(alpha_data)),
+      timestamp(timestamp),
+      is_keyframe(is_keyframe) {}
 
 WebmMuxer::EncodedVideoFrame::~EncodedVideoFrame() {}
 
 }  // namespace media