[EXPERIMENTAL] Generic stereo codec with index header sending single frames

media/engine/webrtcvideoengine.cc

Index: media/engine/webrtcvideoengine.cc
diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc
index 29fc442cb1f2742a88ba882107abec1fa5c88eca..dc3ca66d2b2308156bca8c42f49a0577b8c31436 100644
--- a/media/engine/webrtcvideoengine.cc
+++ b/media/engine/webrtcvideoengine.cc
@@ -24,6 +24,7 @@
 #include "api/video_codecs/video_encoder_factory.h"
 #include "call/call.h"
 #include "common_video/h264/profile_level_id.h"
+#include "media/base/codec.h"
 #include "media/engine/constants.h"
 #include "media/engine/internaldecoderfactory.h"
 #include "media/engine/internalencoderfactory.h"
@@ -36,6 +37,8 @@
 #include "media/engine/webrtcmediaengine.h"
 #include "media/engine/webrtcvideoencoderfactory.h"
 #include "media/engine/webrtcvoiceengine.h"
+#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h"
+#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h"
 #include "rtc_base/copyonwritebuffer.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/stringutils.h"
@@ -68,7 +71,8 @@ class EncoderFactoryAdapter {
 
   virtual AllocatedEncoder CreateVideoEncoder(
       const VideoCodec& codec,
-      bool is_conference_mode_screenshare) const = 0;
+      bool is_conference_mode_screenshare,
+      bool is_stereo_codec) const = 0;
 
   virtual std::vector<VideoCodec> GetSupportedCodecs() const = 0;
 };
@@ -79,7 +83,8 @@ class DecoderFactoryAdapter {
 
   virtual std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(
       const VideoCodec& codec,
-      const VideoDecoderParams& decoder_params) const = 0;
+      const VideoDecoderParams& decoder_params,
+      bool is_stereo_codec) const = 0;
 };
 
 namespace {
@@ -105,12 +110,14 @@ class CricketEncoderFactoryAdapter : public EncoderFactoryAdapter {
 
   AllocatedEncoder CreateVideoEncoder(
       const VideoCodec& codec,
-      bool is_conference_mode_screenshare) const override;
+      bool is_conference_mode_screenshare,
+      bool is_stereo_codec) const override;
 
   std::vector<VideoCodec> GetSupportedCodecs() const override;
 
   const std::unique_ptr<WebRtcVideoEncoderFactory> internal_encoder_factory_;
   WebRtcVideoEncoderFactory* const external_encoder_factory_;
+  std::unique_ptr<WebRtcVideoEncoderFactory> stereo_encoder_factory_;
 };
 
 class CricketDecoderFactoryAdapter : public DecoderFactoryAdapter {
@@ -127,10 +134,12 @@ class CricketDecoderFactoryAdapter : public DecoderFactoryAdapter {
 
   std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(
       const VideoCodec& codec,
-      const VideoDecoderParams& decoder_params) const override;
+      const VideoDecoderParams& decoder_params,
+      bool is_stereo_codec) const override;
 
   const std::unique_ptr<WebRtcVideoDecoderFactory> internal_decoder_factory_;
   WebRtcVideoDecoderFactory* const external_decoder_factory_;
+  std::unique_ptr<WebRtcVideoDecoderFactory> stereo_decoder_factory_;
 };
 
 // Wraps webrtc::VideoEncoderFactory into common EncoderFactoryAdapter
@@ -144,7 +153,8 @@ class WebRtcEncoderFactoryAdapter : public EncoderFactoryAdapter {
  private:
   AllocatedEncoder CreateVideoEncoder(
       const VideoCodec& codec,
-      bool is_conference_mode_screenshare) const override {
+      bool is_conference_mode_screenshare,
+      bool is_stereo_codec) const override {
     if (!encoder_factory_)
       return AllocatedEncoder();
     const webrtc::SdpVideoFormat format(codec.name, codec.params);
@@ -183,7 +193,8 @@ class WebRtcDecoderFactoryAdapter : public DecoderFactoryAdapter {
  private:
   std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(
       const VideoCodec& codec,
-      const VideoDecoderParams& decoder_params) const override {
+      const VideoDecoderParams& decoder_params,
+      bool is_stereo_codec) const override {
     return decoder_factory_
                ? decoder_factory_->CreateVideoDecoder(
                      webrtc::SdpVideoFormat(codec.name, codec.params))
@@ -556,6 +567,14 @@ std::vector<VideoCodec> AssignPayloadTypesAndAddAssociatedRtxCodecs(
       if (payload_type > kLastDynamicPayloadType)
         break;
     }
+
+    if (CodecNamesEq(codec.name, kVp9CodecName)) {
+      output_codecs.push_back(
+          VideoCodec::CreateStereoCodec(payload_type, codec));
+      ++payload_type;
+      if (payload_type > kLastDynamicPayloadType)
+        break;
+    }
   }
   return output_codecs;
 }
@@ -583,12 +602,11 @@ std::vector<VideoCodec> CricketEncoderFactoryAdapter::GetSupportedCodecs()
   return AssignPayloadTypesAndAddAssociatedRtxCodecs(codecs);
 }
 
-WebRtcVideoChannel::WebRtcVideoChannel(
-    webrtc::Call* call,
-    const MediaConfig& config,
-    const VideoOptions& options,
-    const EncoderFactoryAdapter* encoder_factory,
-    const DecoderFactoryAdapter* decoder_factory)
+WebRtcVideoChannel::WebRtcVideoChannel(webrtc::Call* call,
+                                       const MediaConfig& config,
+                                       const VideoOptions& options,
+                                       EncoderFactoryAdapter* encoder_factory,
+                                       DecoderFactoryAdapter* decoder_factory)
     : VideoMediaChannel(config),
       call_(call),
       unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_),
@@ -619,6 +637,9 @@ WebRtcVideoChannel::SelectSendVideoCodec(
       encoder_factory_->GetSupportedCodecs();
   // Select the first remote codec that is supported locally.
   for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
+    // HARDCODE TO ALPHA
+    if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec))
+      continue;
     // For H264, we will limit the encode level to the remote offered level
     // regardless if level asymmetry is allowed or not. This is strictly not
     // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
@@ -631,6 +652,24 @@ WebRtcVideoChannel::SelectSendVideoCodec(
   return rtc::Optional<VideoCodecSettings>();
 }
 
+rtc::Optional<WebRtcVideoChannel::VideoCodecSettings>
+WebRtcVideoChannel::SelectStereoAssociatedVideoCodec(
+    const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
+  const std::vector<VideoCodec> local_supported_codecs =
+      encoder_factory_->GetSupportedCodecs();
+
+  // Select the first remote codec that is supported locally.
+  for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
+    // HARDCODE TO VP9
+    if (!CodecNamesEq(remote_mapped_codec.codec.name.c_str(), kVp9CodecName))
+      continue;
+    if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec) &&
+        FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec))
+      return rtc::Optional<VideoCodecSettings>(remote_mapped_codec);
+  }
+  return rtc::Optional<VideoCodecSettings>();
+}
+
 bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged(
     std::vector<VideoCodecSettings> before,
     std::vector<VideoCodecSettings> after) {
@@ -677,6 +716,19 @@ bool WebRtcVideoChannel::GetChangedSendParameters(
     return false;
   }
 
+  if (VideoCodec::IsStereoCodec(selected_send_codec->codec)) {
+    rtc::Optional<VideoCodecSettings> associated_codec_settings =
+        SelectStereoAssociatedVideoCodec(MapCodecs(params.codecs));
+    LOG(LS_ERROR) << __func__ << associated_codec_settings->codec.ToString();
+    if (!associated_codec_settings) {
+      LOG(LS_ERROR)
+          << "Stereo codec is not associated with any supported codec.";
+      return false;
+    }
+    associated_codec_settings->stereo_codec.emplace(selected_send_codec->codec);
+    selected_send_codec = associated_codec_settings;
+  }
+
   // Never enable sending FlexFEC, unless we are in the experiment.
   if (!IsFlexfecFieldTrialEnabled()) {
     if (selected_send_codec->flexfec_payload_type != -1) {
@@ -1549,7 +1601,7 @@ WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream(
     const StreamParams& sp,
     webrtc::VideoSendStream::Config config,
     const VideoOptions& options,
-    const EncoderFactoryAdapter* encoder_factory,
+    EncoderFactoryAdapter* encoder_factory,
     bool enable_cpu_overuse_detection,
     int max_bitrate_bps,
     const rtc::Optional<VideoCodecSettings>& codec_settings,
@@ -1699,7 +1751,30 @@ WebRtcVideoChannel::WebRtcVideoSendStream::GetSsrcs() const {
 EncoderFactoryAdapter::AllocatedEncoder
 CricketEncoderFactoryAdapter::CreateVideoEncoder(
     const VideoCodec& codec,
-    bool is_conference_mode_screenshare) const {
+    bool is_conference_mode_screenshare,
+    bool is_stereo_codec) const {
+  if (is_stereo_codec) {
+    if (external_encoder_factory_ != nullptr &&
+        FindMatchingCodec(external_encoder_factory_->supported_codecs(),
+                          codec)) {
+      std::unique_ptr<webrtc::VideoEncoder> stereo_encoder(
+          new webrtc::StereoEncoderAdapter(external_encoder_factory_));
+      return AllocatedEncoder(std::move(stereo_encoder),
+                              true /* is_hardware_accelerated */,
+                              false /* is_external */);
+    }
+    if (FindMatchingCodec(internal_encoder_factory_->supported_codecs(),
+                          codec)) {
+      std::unique_ptr<webrtc::VideoEncoder> stereo_encoder(
+          new webrtc::StereoEncoderAdapter(internal_encoder_factory_.get()));
+      return AllocatedEncoder(std::move(stereo_encoder),
+                              false /* is_hardware_accelerated */,
+                              false /* is_external */);
+    }
+    RTC_NOTREACHED();
+    return AllocatedEncoder();
+  }
+
   // Try creating external encoder.
   if (external_encoder_factory_ != nullptr &&
       FindMatchingCodec(external_encoder_factory_->supported_codecs(), codec)) {
@@ -1757,6 +1832,8 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
   parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec);
   RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
 
+  const bool is_stereo_codec = codec_settings.stereo_codec.has_value();
+
   // Do not re-create encoders of the same type. We can't overwrite
   // |allocated_encoder_| immediately, because we need to release it after the
   // RecreateWebRtcStream() call.
@@ -1769,7 +1846,8 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
         parameters_.conference_mode;
     EncoderFactoryAdapter::AllocatedEncoder new_allocated_encoder =
         encoder_factory_->CreateVideoEncoder(codec_settings.codec,
-                                             is_conference_mode_screenshare);
+                                             is_conference_mode_screenshare,
+                                             is_stereo_codec);
     new_encoder = std::unique_ptr<webrtc::VideoEncoder>(
         std::move(new_allocated_encoder.encoder));
     parameters_.config.encoder_settings.encoder = new_encoder.get();
@@ -1780,8 +1858,13 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
   } else {
     new_encoder = std::move(allocated_encoder_);
   }
-  parameters_.config.encoder_settings.payload_name = codec_settings.codec.name;
-  parameters_.config.encoder_settings.payload_type = codec_settings.codec.id;
+  VideoCodec payload_codec = is_stereo_codec
+                                 ? codec_settings.stereo_codec.value()
+                                 : codec_settings.codec;
+  parameters_.config.encoder_settings.payload_name = payload_codec.name;
+  parameters_.config.encoder_settings.payload_type = payload_codec.id;
+  parameters_.config.encoder_settings.stereo_associated_payload_name =
+      codec_settings.codec.name;
   parameters_.config.rtp.ulpfec = codec_settings.ulpfec;
   parameters_.config.rtp.flexfec.payload_type =
       codec_settings.flexfec_payload_type;
@@ -2024,6 +2107,7 @@ VideoSenderInfo WebRtcVideoChannel::WebRtcVideoSendStream::GetVideoSenderInfo(
   for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
     info.add_ssrc(ssrc);
 
+  // TODO(emircan): Add stereo codec case.
   if (parameters_.codec_settings) {
     info.codec_name = parameters_.codec_settings->codec.name;
     info.codec_payload_type = rtc::Optional<int>(
@@ -2150,7 +2234,7 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
     webrtc::Call* call,
     const StreamParams& sp,
     webrtc::VideoReceiveStream::Config config,
-    const DecoderFactoryAdapter* decoder_factory,
+    DecoderFactoryAdapter* decoder_factory,
     bool default_stream,
     const std::vector<VideoCodecSettings>& recv_codecs,
     const webrtc::FlexfecReceiveStream::Config& flexfec_config)
@@ -2204,7 +2288,19 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetFirstPrimarySsrc() const {
 std::unique_ptr<webrtc::VideoDecoder>
 CricketDecoderFactoryAdapter::CreateVideoDecoder(
     const VideoCodec& codec,
-    const VideoDecoderParams& decoder_params) const {
+    const VideoDecoderParams& decoder_params,
+    bool is_stereo_codec) const {
+  if (is_stereo_codec) {
+    if (external_decoder_factory_ != nullptr) {
+      std::unique_ptr<webrtc::VideoDecoder> external_decoder(
+          new webrtc::StereoDecoderAdapter(external_decoder_factory_));
+      return external_decoder;
+    }
+    std::unique_ptr<webrtc::VideoDecoder> internal_decoder(
+        new webrtc::StereoDecoderAdapter(internal_decoder_factory_.get()));
+    return internal_decoder;
+  }
+
   if (external_decoder_factory_ != nullptr) {
     std::unique_ptr<webrtc::VideoDecoder> external_decoder =
         CreateScopedVideoDecoder(external_decoder_factory_, codec,
@@ -2232,19 +2328,21 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs(
   config_.decoders.clear();
   config_.rtp.rtx_associated_payload_types.clear();
   for (const auto& recv_codec : recv_codecs) {
+    const bool is_stereo_codec =
+        cricket::VideoCodec::IsStereoCodec(recv_codec.codec);
     webrtc::SdpVideoFormat video_format(recv_codec.codec.name,
                                         recv_codec.codec.params);
     std::unique_ptr<webrtc::VideoDecoder> new_decoder;
 
     auto it = old_decoders->find(video_format);
-    if (it != old_decoders->end()) {
+    if (it != old_decoders->end() && !is_stereo_codec) {
       new_decoder = std::move(it->second);
       old_decoders->erase(it);
     }
 
     if (!new_decoder) {
-      new_decoder = decoder_factory_->CreateVideoDecoder(recv_codec.codec,
-                                                         {stream_params_.id});
+      new_decoder = decoder_factory_->CreateVideoDecoder(
+          recv_codec.codec, {stream_params_.id}, is_stereo_codec);
     }
 
     webrtc::VideoReceiveStream::Decoder decoder;