[EXPERIMENTAL] Generic stereo codec with index header sending merged frames

webrtc/media/engine/webrtcvideoengine.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/media/engine/webrtcvideoengine.h"
 
 #include <stdio.h>
 #include <algorithm>
 #include <set>
 #include <string>
 #include <utility>
 
 #include "webrtc/api/video/i420_buffer.h"
 #include "webrtc/api/video_codecs/video_decoder.h"
 #include "webrtc/api/video_codecs/video_encoder.h"
 #include "webrtc/call/call.h"
 #include "webrtc/common_video/h264/profile_level_id.h"
+#include "webrtc/media/base/codec.h"
 #include "webrtc/media/engine/constants.h"
 #include "webrtc/media/engine/internaldecoderfactory.h"
 #include "webrtc/media/engine/internalencoderfactory.h"
 #include "webrtc/media/engine/simulcast.h"
 #include "webrtc/media/engine/simulcast_encoder_adapter.h"
 #include "webrtc/media/engine/videodecodersoftwarefallbackwrapper.h"
 #include "webrtc/media/engine/videoencodersoftwarefallbackwrapper.h"
 #include "webrtc/media/engine/webrtcmediaengine.h"
 #include "webrtc/media/engine/webrtcvideoencoderfactory.h"
 #include "webrtc/media/engine/webrtcvoiceengine.h"
+#include "webrtc/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h"
+#include "webrtc/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h"
 #include "webrtc/rtc_base/copyonwritebuffer.h"
 #include "webrtc/rtc_base/logging.h"
 #include "webrtc/rtc_base/stringutils.h"
 #include "webrtc/rtc_base/timeutils.h"
 #include "webrtc/rtc_base/trace_event.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 
 using DegradationPreference = webrtc::VideoSendStream::DegradationPreference;
 
 namespace cricket {
@@ skipping 11 matching lines @@
   return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03-Advertised");
 }
 
 // If this field trial is enabled, we will report VideoContentType RTP extension
 // in capabilities (thus, it will end up in the default SDP and extension will
 // be sent for all key-frames).
 bool IsVideoContentTypeExtensionFieldTrialEnabled() {
   return webrtc::field_trial::IsEnabled("WebRTC-VideoContentTypeExtension");
 }
 
+// Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory.
+class EncoderFactoryAdapter : public webrtc::VideoEncoderFactory {
+ public:
+  // EncoderFactoryAdapter doesn't take ownership of |factory|, which is owned
+  // by e.g. PeerConnectionFactory.
+  explicit EncoderFactoryAdapter(cricket::WebRtcVideoEncoderFactory* factory,
+                                 const cricket::VideoCodec& codec)
+      : factory_(factory), codec_(codec) {}
+  virtual ~EncoderFactoryAdapter() {}
+
+  // Implement webrtc::VideoEncoderFactory.
+  webrtc::VideoEncoder* Create() override {
+    return factory_->CreateVideoEncoder(codec_);
+  }
+
+  void Destroy(webrtc::VideoEncoder* encoder) override {
+    return factory_->DestroyVideoEncoder(encoder);
+  }
+
+ private:
+  cricket::WebRtcVideoEncoderFactory* const factory_;
+  const cricket::VideoCodec codec_;
+};
+
 // An encoder factory that wraps Create requests for simulcastable codec types
 // with a webrtc::SimulcastEncoderAdapter. Non simulcastable codec type
 // requests are just passed through to the contained encoder factory.
 class WebRtcSimulcastEncoderFactory
     : public cricket::WebRtcVideoEncoderFactory {
  public:
   // WebRtcSimulcastEncoderFactory doesn't take ownership of |factory|, which is
   // owned by e.g. PeerConnectionFactory.
   explicit WebRtcSimulcastEncoderFactory(
       cricket::WebRtcVideoEncoderFactory* factory)
@@ skipping 49 matching lines @@
     delete encoder;
   }
 
  private:
   cricket::WebRtcVideoEncoderFactory* factory_;
   // A list of encoders that were created without being wrapped in a
   // SimulcastEncoderAdapter.
   std::vector<webrtc::VideoEncoder*> non_simulcast_encoders_;
 };
 
+// An encoder factory that wraps Create requests for all codec types
+// with a webrtc::StereoEncoderAdapter.
+class WebRtcStereoEncoderFactory : public cricket::WebRtcVideoEncoderFactory {
+ public:
+  explicit WebRtcStereoEncoderFactory(
+      cricket::WebRtcVideoEncoderFactory* factory)
+      : factory_(factory) {}
+
+  webrtc::VideoEncoder* CreateVideoEncoder(
+      const cricket::VideoCodec& codec) override {
+    RTC_DCHECK(factory_ != NULL);
+    return new webrtc::StereoEncoderAdapter(
+        new EncoderFactoryAdapter(factory_, codec));
+  }
+
+  const std::vector<cricket::VideoCodec>& supported_codecs() const override {
+    return factory_->supported_codecs();
+  }
+
+  bool EncoderTypeHasInternalSource(
+      webrtc::VideoCodecType type) const override {
+    return factory_->EncoderTypeHasInternalSource(type);
+  }
+
+  void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override {
+    delete encoder;
+  }
+
+ private:
+  cricket::WebRtcVideoEncoderFactory* factory_;
+};
+
+// Wrap cricket::WebRtcVideoDecoderFactory as a webrtc::VideoDecoderFactory.
+class DecoderFactoryAdapter : public webrtc::VideoDecoderFactory {
+ public:
+  // DecoderFactoryAdapter doesn't take ownership of |factory|.
+  explicit DecoderFactoryAdapter(cricket::WebRtcVideoDecoderFactory* factory,
+                                 webrtc::VideoCodecType type,
+                                 VideoDecoderParams* params)
+      : factory_(factory), type_(type) {
+    if (params) {
+      params_.reset(new VideoDecoderParams(*params));
+    }
+  }
+  virtual ~DecoderFactoryAdapter() {}
+
+  // Implement webrtc::VideoDecoderFactory.
+  webrtc::VideoDecoder* Create() override {
+    RTC_DCHECK(factory_);
+    if (params_)
+      return factory_->CreateVideoDecoderWithParams(type_, *params_);
+    else
+      return factory_->CreateVideoDecoder(type_);
+  }
+
+  void Destroy(webrtc::VideoDecoder* decoder) override {
+    return factory_->DestroyVideoDecoder(decoder);
+  }
+
+ private:
+  cricket::WebRtcVideoDecoderFactory* const factory_;
+  const webrtc::VideoCodecType type_;
+  std::unique_ptr<VideoDecoderParams> params_;
+};
+
+// A decoder factory that wraps Create requests for all codec types
+// with a webrtc::StereoDecoderAdapter.
+class WebRtcStereoDecoderFactory : public cricket::WebRtcVideoDecoderFactory {
+ public:
+  explicit WebRtcStereoDecoderFactory(
+      cricket::WebRtcVideoDecoderFactory* factory)
+      : factory_(factory) {}
+
+  webrtc::VideoDecoder* CreateVideoDecoder(
+      webrtc::VideoCodecType type) override {
+    RTC_DCHECK(factory_ != NULL);
+    return new webrtc::StereoDecoderAdapter(
+        new DecoderFactoryAdapter(factory_, webrtc::kVideoCodecVP9, nullptr));
+  }
+
+  webrtc::VideoDecoder* CreateVideoDecoderWithParams(
+      webrtc::VideoCodecType type,
+      VideoDecoderParams params) override {
+    RTC_DCHECK(factory_ != NULL);
+    return new webrtc::StereoDecoderAdapter(
+        new DecoderFactoryAdapter(factory_, webrtc::kVideoCodecVP9, &params));
+  }
+
+  void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override {
+    delete decoder;
+  }
+
+ private:
+  cricket::WebRtcVideoDecoderFactory* factory_;
+};
+
 void AddDefaultFeedbackParams(VideoCodec* codec) {
   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir));
   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty));
   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli));
   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty));
   codec->AddFeedbackParam(
       FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty));
 }
 
 static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) {
@@ skipping 375 matching lines @@
         CodecNamesEq(codec.name, kVp9CodecName) ||
         CodecNamesEq(codec.name, kH264CodecName) ||
         CodecNamesEq(codec.name, kRedCodecName)) {
       const rtc::Optional<int> rtx_payload_type =
           NextFreePayloadType(*unified_codecs);
       if (!rtx_payload_type)
         return;
       unified_codecs->push_back(
           VideoCodec::CreateRtxCodec(*rtx_payload_type, codec.id));
     }
+
+    if (CodecNamesEq(codec.name, kVp9CodecName)) {
+      const rtc::Optional<int> stereo_payload_type =
+          NextFreePayloadType(*unified_codecs);
+      if (!stereo_payload_type)
+        return;
+      unified_codecs->push_back(
+          VideoCodec::CreateStereoCodec(*stereo_payload_type, codec));
+    }
   }
 }
 
 static std::vector<VideoCodec> GetSupportedCodecs(
     const WebRtcVideoEncoderFactory* external_encoder_factory) {
   const std::vector<VideoCodec> internal_codecs =
       InternalEncoderFactory().supported_codecs();
   LOG(LS_INFO) << "Internally supported codecs: "
                << CodecVectorToString(internal_codecs);
 
@@ skipping 38 matching lines @@
     delete kv.second;
   for (auto& kv : receive_streams_)
     delete kv.second;
 }
 
 rtc::Optional<WebRtcVideoChannel::VideoCodecSettings>
 WebRtcVideoChannel::SelectSendVideoCodec(
     const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
   const std::vector<VideoCodec> local_supported_codecs =
       GetSupportedCodecs(external_encoder_factory_);
+
   // Select the first remote codec that is supported locally.
   for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
+    // HARDCODE TO ALPHA
+    if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec))
+      continue;
     // For H264, we will limit the encode level to the remote offered level
     // regardless if level asymmetry is allowed or not. This is strictly not
     // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
     // since we should limit the encode level to the lower of local and remote
     // level when level asymmetry is not allowed.
     if (FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec))
       return rtc::Optional<VideoCodecSettings>(remote_mapped_codec);
   }
   // No remote codec was supported.
   return rtc::Optional<VideoCodecSettings>();
 }
 
+rtc::Optional<WebRtcVideoChannel::VideoCodecSettings>
+WebRtcVideoChannel::SelectStereoAssociatedVideoCodec(
+    const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
+  const std::vector<VideoCodec> local_supported_codecs =
+      GetSupportedCodecs(external_encoder_factory_);
+
+  // Select the first remote codec that is supported locally.
+  for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
+    // HARDCODE TO VP9
+    if (!CodecNamesEq(remote_mapped_codec.codec.name.c_str(), kVp9CodecName))
+      continue;
+    if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec) &&
+        FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec))
+      return rtc::Optional<VideoCodecSettings>(remote_mapped_codec);
+  }
+  return rtc::Optional<VideoCodecSettings>();
+}
+
 bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged(
     std::vector<VideoCodecSettings> before,
     std::vector<VideoCodecSettings> after) {
   if (before.size() != after.size()) {
     return true;
   }
 
   // The receive codec order doesn't matter, so we sort the codecs before
   // comparing. This is necessary because currently the
   // only way to change the send codec is to munge SDP, which causes
@@ skipping 26 matching lines @@
 
   // Select one of the remote codecs that will be used as send codec.
   rtc::Optional<VideoCodecSettings> selected_send_codec =
       SelectSendVideoCodec(MapCodecs(params.codecs));
 
   if (!selected_send_codec) {
     LOG(LS_ERROR) << "No video codecs supported.";
     return false;
   }
 
+  if (VideoCodec::IsStereoCodec(selected_send_codec->codec)) {
+    rtc::Optional<VideoCodecSettings> associated_codec_settings =
+        SelectStereoAssociatedVideoCodec(MapCodecs(params.codecs));
+    LOG(LS_ERROR) << __func__ << associated_codec_settings->codec.ToString();
+    if (!associated_codec_settings) {
+      LOG(LS_ERROR)
+          << "Stereo codec is not associated with any supported codec.";
+      return false;
+    }
+    associated_codec_settings->stereo_codec.emplace(selected_send_codec->codec);
+    selected_send_codec = associated_codec_settings;
+  }
+
   // Never enable sending FlexFEC, unless we are in the experiment.
   if (!IsFlexfecFieldTrialEnabled()) {
     if (selected_send_codec->flexfec_payload_type != -1) {
       LOG(LS_INFO) << "Remote supports flexfec-03, but we will not send since "
                    << "WebRTC-FlexFEC-03 field trial is not enabled.";
     }
     selected_send_codec->flexfec_payload_type = -1;
   }
 
   if (!send_codec_ || *selected_send_codec != *send_codec_)
@@ skipping 1008 matching lines @@
   }
   return degradation_preference;
 }
 
 const std::vector<uint32_t>&
 WebRtcVideoChannel::WebRtcVideoSendStream::GetSsrcs() const {
   return ssrcs_;
 }
 
 WebRtcVideoChannel::WebRtcVideoSendStream::AllocatedEncoder
+WebRtcVideoChannel::WebRtcVideoSendStream::CreateStereoVideoEncoder(
+    const VideoCodec& codec) {
+  WebRtcStereoEncoderFactory* stereo_factory =
+      new WebRtcStereoEncoderFactory(internal_encoder_factory_.get());
+  stereo_encoder_factory_.reset(stereo_factory);
+  return AllocatedEncoder(stereo_factory->CreateVideoEncoder(codec), codec,
+                          false /* is_external */);
+}
+
+WebRtcVideoChannel::WebRtcVideoSendStream::AllocatedEncoder
 WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoder(
     const VideoCodec& codec,
     bool force_encoder_allocation) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   // Do not re-create encoders of the same type.
   if (!force_encoder_allocation && codec == allocated_encoder_.codec &&
       allocated_encoder_.encoder != nullptr) {
     return allocated_encoder_;
   }
 
@@ skipping 38 matching lines @@
   delete encoder->encoder;
 }
 
 void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
     const VideoCodecSettings& codec_settings,
     bool force_encoder_allocation) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec);
   RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
 
+  const bool is_stereo_codec = codec_settings.stereo_codec.has_value();
   AllocatedEncoder new_encoder =
-      CreateVideoEncoder(codec_settings.codec, force_encoder_allocation);
+      is_stereo_codec
+          ? CreateStereoVideoEncoder(codec_settings.codec)
+          : CreateVideoEncoder(codec_settings.codec, force_encoder_allocation);
+  VideoCodec payload_codec = is_stereo_codec
+                                 ? codec_settings.stereo_codec.value()
+                                 : codec_settings.codec;
+
   parameters_.config.encoder_settings.encoder = new_encoder.encoder;
+  parameters_.config.encoder_settings.payload_name = payload_codec.name;
+  parameters_.config.encoder_settings.payload_type = payload_codec.id;
+  parameters_.config.encoder_settings.stereo_associated_payload_name =
+      codec_settings.codec.name;
   parameters_.config.encoder_settings.full_overuse_time = new_encoder.external;
-  parameters_.config.encoder_settings.payload_name = codec_settings.codec.name;
-  parameters_.config.encoder_settings.payload_type = codec_settings.codec.id;
   if (new_encoder.external) {
     webrtc::VideoCodecType type =
         webrtc::PayloadNameToCodecType(codec_settings.codec.name)
             .value_or(webrtc::kVideoCodecUnknown);
     parameters_.config.encoder_settings.internal_source =
         external_encoder_factory_->EncoderTypeHasInternalSource(type);
   } else {
     parameters_.config.encoder_settings.internal_source = false;
   }
   parameters_.config.rtp.ulpfec = codec_settings.ulpfec;
@@ skipping 233 matching lines @@
   }
 }
 
 VideoSenderInfo WebRtcVideoChannel::WebRtcVideoSendStream::GetVideoSenderInfo(
     bool log_stats) {
   VideoSenderInfo info;
   RTC_DCHECK_RUN_ON(&thread_checker_);
   for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
     info.add_ssrc(ssrc);
 
+  // TODO(emircan): Add stereo codec case.
   if (parameters_.codec_settings) {
     info.codec_name = parameters_.codec_settings->codec.name;
     info.codec_payload_type = rtc::Optional<int>(
         parameters_.codec_settings->codec.id);
   }
 
   if (stream_ == NULL)
     return info;
 
   webrtc::VideoSendStream::Stats stats = stream_->GetStats();
@@ skipping 116 matching lines @@
     const std::vector<VideoCodecSettings>& recv_codecs,
     const webrtc::FlexfecReceiveStream::Config& flexfec_config)
     : call_(call),
       stream_params_(sp),
       stream_(NULL),
       default_stream_(default_stream),
       config_(std::move(config)),
       flexfec_config_(flexfec_config),
       flexfec_stream_(nullptr),
       external_decoder_factory_(external_decoder_factory),
+      internal_decoder_factory_(new InternalDecoderFactory()),
       sink_(NULL),
       first_frame_timestamp_(-1),
       estimated_remote_start_ntp_time_ms_(0) {
   config_.renderer = this;
   std::vector<AllocatedDecoder> old_decoders;
   ConfigureCodecs(recv_codecs, &old_decoders);
   ConfigureFlexfecCodec(flexfec_config.payload_type);
   MaybeRecreateWebRtcFlexfecStream();
   RecreateWebRtcVideoStream();
   RTC_DCHECK(old_decoders.empty());
@@ skipping 34 matching lines @@
 
   if (primary_ssrcs.empty()) {
     LOG(LS_WARNING) << "Empty primary ssrcs vector, returning empty optional";
     return rtc::Optional<uint32_t>();
   } else {
     return rtc::Optional<uint32_t>(primary_ssrcs[0]);
   }
 }
 
 WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder
+WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateStereoVideoDecoder(
+    const VideoCodec& codec) {
+  LOG(LS_ERROR) << __func__;
+  webrtc::VideoCodecType type = webrtc::PayloadNameToCodecType(codec.name)
+                                    .value_or(webrtc::kVideoCodecUnknown);
+  stereo_decoder_factory_.reset(
+      new WebRtcStereoDecoderFactory(internal_decoder_factory_.get()));
+  return AllocatedDecoder(stereo_decoder_factory_->CreateVideoDecoderWithParams(
+                              type, {stream_params_.id}),
+                          type, false /* is_external */);
+}
+
+WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder
 WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder(
     std::vector<AllocatedDecoder>* old_decoders,
     const VideoCodec& codec) {
   webrtc::VideoCodecType type = webrtc::PayloadNameToCodecType(codec.name)
                                     .value_or(webrtc::kVideoCodecUnknown);
 
   for (size_t i = 0; i < old_decoders->size(); ++i) {
     if ((*old_decoders)[i].type == type) {
       AllocatedDecoder decoder = (*old_decoders)[i];
       (*old_decoders)[i] = old_decoders->back();
@@ skipping 17 matching lines @@
                           type, false /* is_external */);
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs(
     const std::vector<VideoCodecSettings>& recv_codecs,
     std::vector<AllocatedDecoder>* old_decoders) {
   *old_decoders = allocated_decoders_;
   allocated_decoders_.clear();
   config_.decoders.clear();
   for (size_t i = 0; i < recv_codecs.size(); ++i) {
+    const bool is_stereo_codec =
+        cricket::VideoCodec::IsStereoCodec(recv_codecs[i].codec);
     AllocatedDecoder allocated_decoder =
-        CreateOrReuseVideoDecoder(old_decoders, recv_codecs[i].codec);
+        is_stereo_codec
+            ? CreateStereoVideoDecoder(recv_codecs[i].codec)
+            : CreateOrReuseVideoDecoder(old_decoders, recv_codecs[i].codec);
     allocated_decoders_.push_back(allocated_decoder);
 
     webrtc::VideoReceiveStream::Decoder decoder;
     decoder.decoder = allocated_decoder.decoder;
     decoder.payload_type = recv_codecs[i].codec.id;
     decoder.payload_name = recv_codecs[i].codec.name;
     decoder.codec_params = recv_codecs[i].codec.params;
     config_.decoders.push_back(decoder);
   }
 
@@ skipping 410 matching lines @@
     stream.temporal_layer_thresholds_bps.resize(GetDefaultVp9TemporalLayers() -
                                                 1);
   }
 
   std::vector<webrtc::VideoStream> streams;
   streams.push_back(stream);
   return streams;
 }
 
 }  // namespace cricket