[EXPERIMENTAL] Generic stereo codec with index header sending merged frames

webrtc/media/engine/webrtcvideoengine.cc

Index: webrtc/media/engine/webrtcvideoengine.cc
diff --git a/webrtc/media/engine/webrtcvideoengine.cc b/webrtc/media/engine/webrtcvideoengine.cc
index b6447329e265519da1f19d29f1227006feeff8c0..e38bc81610d3349fe0f440f3f57fbf383c0bc68d 100644
--- a/webrtc/media/engine/webrtcvideoengine.cc
+++ b/webrtc/media/engine/webrtcvideoengine.cc
@@ -21,6 +21,7 @@
 #include "webrtc/api/video_codecs/video_encoder.h"
 #include "webrtc/call/call.h"
 #include "webrtc/common_video/h264/profile_level_id.h"
+#include "webrtc/media/base/codec.h"
 #include "webrtc/media/engine/constants.h"
 #include "webrtc/media/engine/internaldecoderfactory.h"
 #include "webrtc/media/engine/internalencoderfactory.h"
@@ -31,6 +32,8 @@
 #include "webrtc/media/engine/webrtcmediaengine.h"
 #include "webrtc/media/engine/webrtcvideoencoderfactory.h"
 #include "webrtc/media/engine/webrtcvoiceengine.h"
+#include "webrtc/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h"
+#include "webrtc/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h"
 #include "webrtc/rtc_base/copyonwritebuffer.h"
 #include "webrtc/rtc_base/logging.h"
 #include "webrtc/rtc_base/stringutils.h"
@@ -62,6 +65,30 @@ bool IsVideoContentTypeExtensionFieldTrialEnabled() {
   return webrtc::field_trial::IsEnabled("WebRTC-VideoContentTypeExtension");
 }
 
+// Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory.
+class EncoderFactoryAdapter : public webrtc::VideoEncoderFactory {
+ public:
+  // EncoderFactoryAdapter doesn't take ownership of |factory|, which is owned
+  // by e.g. PeerConnectionFactory.
+  explicit EncoderFactoryAdapter(cricket::WebRtcVideoEncoderFactory* factory,
+                                 const cricket::VideoCodec& codec)
+      : factory_(factory), codec_(codec) {}
+  virtual ~EncoderFactoryAdapter() {}
+
+  // Implement webrtc::VideoEncoderFactory.
+  webrtc::VideoEncoder* Create() override {
+    return factory_->CreateVideoEncoder(codec_);
+  }
+
+  void Destroy(webrtc::VideoEncoder* encoder) override {
+    return factory_->DestroyVideoEncoder(encoder);
+  }
+
+ private:
+  cricket::WebRtcVideoEncoderFactory* const factory_;
+  const cricket::VideoCodec codec_;
+};
+
 // An encoder factory that wraps Create requests for simulcastable codec types
 // with a webrtc::SimulcastEncoderAdapter. Non simulcastable codec type
 // requests are just passed through to the contained encoder factory.
@@ -131,6 +158,102 @@ class WebRtcSimulcastEncoderFactory
   std::vector<webrtc::VideoEncoder*> non_simulcast_encoders_;
 };
 
+// An encoder factory that wraps Create requests for all codec types
+// with a webrtc::StereoEncoderAdapter.
+class WebRtcStereoEncoderFactory : public cricket::WebRtcVideoEncoderFactory {
+ public:
+  explicit WebRtcStereoEncoderFactory(
+      cricket::WebRtcVideoEncoderFactory* factory)
+      : factory_(factory) {}
+
+  webrtc::VideoEncoder* CreateVideoEncoder(
+      const cricket::VideoCodec& codec) override {
+    RTC_DCHECK(factory_ != NULL);
+    return new webrtc::StereoEncoderAdapter(
+        new EncoderFactoryAdapter(factory_, codec));
+  }
+
+  const std::vector<cricket::VideoCodec>& supported_codecs() const override {
+    return factory_->supported_codecs();
+  }
+
+  bool EncoderTypeHasInternalSource(
+      webrtc::VideoCodecType type) const override {
+    return factory_->EncoderTypeHasInternalSource(type);
+  }
+
+  void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override {
+    delete encoder;
+  }
+
+ private:
+  cricket::WebRtcVideoEncoderFactory* factory_;
+};
+
+// Wrap cricket::WebRtcVideoDecoderFactory as a webrtc::VideoDecoderFactory.
+class DecoderFactoryAdapter : public webrtc::VideoDecoderFactory {
+ public:
+  // DecoderFactoryAdapter doesn't take ownership of |factory|.
+  explicit DecoderFactoryAdapter(cricket::WebRtcVideoDecoderFactory* factory,
+                                 webrtc::VideoCodecType type,
+                                 VideoDecoderParams* params)
+      : factory_(factory), type_(type) {
+    if (params) {
+      params_.reset(new VideoDecoderParams(*params));
+    }
+  }
+  virtual ~DecoderFactoryAdapter() {}
+
+  // Implement webrtc::VideoDecoderFactory.
+  webrtc::VideoDecoder* Create() override {
+    RTC_DCHECK(factory_);
+    if (params_)
+      return factory_->CreateVideoDecoderWithParams(type_, *params_);
+    else
+      return factory_->CreateVideoDecoder(type_);
+  }
+
+  void Destroy(webrtc::VideoDecoder* decoder) override {
+    return factory_->DestroyVideoDecoder(decoder);
+  }
+
+ private:
+  cricket::WebRtcVideoDecoderFactory* const factory_;
+  const webrtc::VideoCodecType type_;
+  std::unique_ptr<VideoDecoderParams> params_;
+};
+
+// A decoder factory that wraps Create requests for all codec types
+// with a webrtc::StereoDecoderAdapter.
+class WebRtcStereoDecoderFactory : public cricket::WebRtcVideoDecoderFactory {
+ public:
+  explicit WebRtcStereoDecoderFactory(
+      cricket::WebRtcVideoDecoderFactory* factory)
+      : factory_(factory) {}
+
+  webrtc::VideoDecoder* CreateVideoDecoder(
+      webrtc::VideoCodecType type) override {
+    RTC_DCHECK(factory_ != NULL);
+    return new webrtc::StereoDecoderAdapter(
+        new DecoderFactoryAdapter(factory_, webrtc::kVideoCodecVP9, nullptr));
+  }
+
+  webrtc::VideoDecoder* CreateVideoDecoderWithParams(
+      webrtc::VideoCodecType type,
+      VideoDecoderParams params) override {
+    RTC_DCHECK(factory_ != NULL);
+    return new webrtc::StereoDecoderAdapter(
+        new DecoderFactoryAdapter(factory_, webrtc::kVideoCodecVP9, &params));
+  }
+
+  void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override {
+    delete decoder;
+  }
+
+ private:
+  cricket::WebRtcVideoDecoderFactory* factory_;
+};
+
 void AddDefaultFeedbackParams(VideoCodec* codec) {
   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir));
   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty));
@@ -526,6 +649,15 @@ static void AppendVideoCodecs(const std::vector<VideoCodec>& input_codecs,
       unified_codecs->push_back(
           VideoCodec::CreateRtxCodec(*rtx_payload_type, codec.id));
     }
+
+    if (CodecNamesEq(codec.name, kVp9CodecName)) {
+      const rtc::Optional<int> stereo_payload_type =
+          NextFreePayloadType(*unified_codecs);
+      if (!stereo_payload_type)
+        return;
+      unified_codecs->push_back(
+          VideoCodec::CreateStereoCodec(*stereo_payload_type, codec));
+    }
   }
 }
 
@@ -584,8 +716,12 @@ WebRtcVideoChannel::SelectSendVideoCodec(
     const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
   const std::vector<VideoCodec> local_supported_codecs =
       GetSupportedCodecs(external_encoder_factory_);
+
   // Select the first remote codec that is supported locally.
   for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
+    // HARDCODE TO ALPHA
+    if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec))
+      continue;
     // For H264, we will limit the encode level to the remote offered level
     // regardless if level asymmetry is allowed or not. This is strictly not
     // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
@@ -598,6 +734,24 @@ WebRtcVideoChannel::SelectSendVideoCodec(
   return rtc::Optional<VideoCodecSettings>();
 }
 
+rtc::Optional<WebRtcVideoChannel::VideoCodecSettings>
+WebRtcVideoChannel::SelectStereoAssociatedVideoCodec(
+    const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
+  const std::vector<VideoCodec> local_supported_codecs =
+      GetSupportedCodecs(external_encoder_factory_);
+
+  // Select the first remote codec that is supported locally.
+  for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
+    // HARDCODE TO VP9
+    if (!CodecNamesEq(remote_mapped_codec.codec.name.c_str(), kVp9CodecName))
+      continue;
+    if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec) &&
+        FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec))
+      return rtc::Optional<VideoCodecSettings>(remote_mapped_codec);
+  }
+  return rtc::Optional<VideoCodecSettings>();
+}
+
 bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged(
     std::vector<VideoCodecSettings> before,
     std::vector<VideoCodecSettings> after) {
@@ -644,6 +798,19 @@ bool WebRtcVideoChannel::GetChangedSendParameters(
     return false;
   }
 
+  if (VideoCodec::IsStereoCodec(selected_send_codec->codec)) {
+    rtc::Optional<VideoCodecSettings> associated_codec_settings =
+        SelectStereoAssociatedVideoCodec(MapCodecs(params.codecs));
+    LOG(LS_ERROR) << __func__ << associated_codec_settings->codec.ToString();
+    if (!associated_codec_settings) {
+      LOG(LS_ERROR)
+          << "Stereo codec is not associated with any supported codec.";
+      return false;
+    }
+    associated_codec_settings->stereo_codec.emplace(selected_send_codec->codec);
+    selected_send_codec = associated_codec_settings;
+  }
+
   // Never enable sending FlexFEC, unless we are in the experiment.
   if (!IsFlexfecFieldTrialEnabled()) {
     if (selected_send_codec->flexfec_payload_type != -1) {
@@ -1672,6 +1839,16 @@ WebRtcVideoChannel::WebRtcVideoSendStream::GetSsrcs() const {
 }
 
 WebRtcVideoChannel::WebRtcVideoSendStream::AllocatedEncoder
+WebRtcVideoChannel::WebRtcVideoSendStream::CreateStereoVideoEncoder(
+    const VideoCodec& codec) {
+  WebRtcStereoEncoderFactory* stereo_factory =
+      new WebRtcStereoEncoderFactory(internal_encoder_factory_.get());
+  stereo_encoder_factory_.reset(stereo_factory);
+  return AllocatedEncoder(stereo_factory->CreateVideoEncoder(codec), codec,
+                          false /* is_external */);
+}
+
+WebRtcVideoChannel::WebRtcVideoSendStream::AllocatedEncoder
 WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoder(
     const VideoCodec& codec,
     bool force_encoder_allocation) {
@@ -1730,12 +1907,21 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
   parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec);
   RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
 
+  const bool is_stereo_codec = codec_settings.stereo_codec.has_value();
   AllocatedEncoder new_encoder =
-      CreateVideoEncoder(codec_settings.codec, force_encoder_allocation);
+      is_stereo_codec
+          ? CreateStereoVideoEncoder(codec_settings.codec)
+          : CreateVideoEncoder(codec_settings.codec, force_encoder_allocation);
+  VideoCodec payload_codec = is_stereo_codec
+                                 ? codec_settings.stereo_codec.value()
+                                 : codec_settings.codec;
+
   parameters_.config.encoder_settings.encoder = new_encoder.encoder;
+  parameters_.config.encoder_settings.payload_name = payload_codec.name;
+  parameters_.config.encoder_settings.payload_type = payload_codec.id;
+  parameters_.config.encoder_settings.stereo_associated_payload_name =
+      codec_settings.codec.name;
   parameters_.config.encoder_settings.full_overuse_time = new_encoder.external;
-  parameters_.config.encoder_settings.payload_name = codec_settings.codec.name;
-  parameters_.config.encoder_settings.payload_type = codec_settings.codec.id;
   if (new_encoder.external) {
     webrtc::VideoCodecType type =
         webrtc::PayloadNameToCodecType(codec_settings.codec.name)
@@ -1989,6 +2175,7 @@ VideoSenderInfo WebRtcVideoChannel::WebRtcVideoSendStream::GetVideoSenderInfo(
   for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
     info.add_ssrc(ssrc);
 
+  // TODO(emircan): Add stereo codec case.
   if (parameters_.codec_settings) {
     info.codec_name = parameters_.codec_settings->codec.name;
     info.codec_payload_type = rtc::Optional<int>(
@@ -2125,6 +2312,7 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
       flexfec_config_(flexfec_config),
       flexfec_stream_(nullptr),
       external_decoder_factory_(external_decoder_factory),
+      internal_decoder_factory_(new InternalDecoderFactory()),
       sink_(NULL),
       first_frame_timestamp_(-1),
       estimated_remote_start_ntp_time_ms_(0) {
@@ -2179,6 +2367,19 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetFirstPrimarySsrc() const {
 }
 
 WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder
+WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateStereoVideoDecoder(
+    const VideoCodec& codec) {
+  LOG(LS_ERROR) << __func__;
+  webrtc::VideoCodecType type = webrtc::PayloadNameToCodecType(codec.name)
+                                    .value_or(webrtc::kVideoCodecUnknown);
+  stereo_decoder_factory_.reset(
+      new WebRtcStereoDecoderFactory(internal_decoder_factory_.get()));
+  return AllocatedDecoder(stereo_decoder_factory_->CreateVideoDecoderWithParams(
+                              type, {stream_params_.id}),
+                          type, false /* is_external */);
+}
+
+WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder
 WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder(
     std::vector<AllocatedDecoder>* old_decoders,
     const VideoCodec& codec) {
@@ -2216,8 +2417,12 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs(
   allocated_decoders_.clear();
   config_.decoders.clear();
   for (size_t i = 0; i < recv_codecs.size(); ++i) {
+    const bool is_stereo_codec =
+        cricket::VideoCodec::IsStereoCodec(recv_codecs[i].codec);
     AllocatedDecoder allocated_decoder =
-        CreateOrReuseVideoDecoder(old_decoders, recv_codecs[i].codec);
+        is_stereo_codec
+            ? CreateStereoVideoDecoder(recv_codecs[i].codec)
+            : CreateOrReuseVideoDecoder(old_decoders, recv_codecs[i].codec);
     allocated_decoders_.push_back(allocated_decoder);
 
     webrtc::VideoReceiveStream::Decoder decoder;