MediaStream audio sourcing: Bypass audio processing for non-WebRTC cases.

content/renderer/media/webrtc/peer_connection_dependency_factory.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webrtc/peer_connection_dependency_factory.h"
 
 #include <stddef.h>
 
 #include <utility>
 #include <vector>
 
 #include "base/command_line.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/metrics/field_trial.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/synchronization/waitable_event.h"
 #include "build/build_config.h"
 #include "content/common/media/media_stream_messages.h"
 #include "content/public/common/content_client.h"
 #include "content/public/common/content_switches.h"
 #include "content/public/common/renderer_preferences.h"
 #include "content/public/common/webrtc_ip_handling_policy.h"
 #include "content/public/renderer/content_renderer_client.h"
 #include "content/renderer/media/media_stream.h"
 #include "content/renderer/media/media_stream_audio_processor.h"
-#include "content/renderer/media/media_stream_audio_processor_options.h"
-#include "content/renderer/media/media_stream_audio_source.h"
 #include "content/renderer/media/media_stream_video_source.h"
 #include "content/renderer/media/media_stream_video_track.h"
 #include "content/renderer/media/peer_connection_identity_store.h"
 #include "content/renderer/media/rtc_media_constraints.h"
 #include "content/renderer/media/rtc_peer_connection_handler.h"
 #include "content/renderer/media/rtc_video_decoder_factory.h"
 #include "content/renderer/media/rtc_video_encoder_factory.h"
-#include "content/renderer/media/webaudio_capturer_source.h"
-#include "content/renderer/media/webrtc/media_stream_remote_audio_track.h"
 #include "content/renderer/media/webrtc/stun_field_trial.h"
-#include "content/renderer/media/webrtc/webrtc_local_audio_track_adapter.h"
 #include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h"
 #include "content/renderer/media/webrtc_audio_device_impl.h"
-#include "content/renderer/media/webrtc_local_audio_track.h"
-#include "content/renderer/media/webrtc_logging.h"
 #include "content/renderer/media/webrtc_uma_histograms.h"
 #include "content/renderer/p2p/empty_network_manager.h"
 #include "content/renderer/p2p/filtering_network_manager.h"
 #include "content/renderer/p2p/ipc_network_manager.h"
 #include "content/renderer/p2p/ipc_socket_factory.h"
 #include "content/renderer/p2p/port_allocator.h"
 #include "content/renderer/render_frame_impl.h"
 #include "content/renderer/render_thread_impl.h"
 #include "content/renderer/render_view_impl.h"
 #include "content/renderer/renderer_features.h"
@@ skipping 34 matching lines @@
     return DEFAULT_PUBLIC_AND_PRIVATE_INTERFACES;
   if (preference == kWebRTCIPHandlingDefaultPublicInterfaceOnly)
     return DEFAULT_PUBLIC_INTERFACE_ONLY;
   if (preference == kWebRTCIPHandlingDisableNonProxiedUdp)
     return DISABLE_NON_PROXIED_UDP;
   return DEFAULT;
 }
 
 }  // namespace
 
-// Map of corresponding media constraints and platform effects.
-struct {
-  const char* constraint;
-  const media::AudioParameters::PlatformEffectsMask effect;
-} const kConstraintEffectMap[] = {
-  { webrtc::MediaConstraintsInterface::kGoogEchoCancellation,
-    media::AudioParameters::ECHO_CANCELLER },
-};
-
-// If any platform effects are available, check them against the constraints.
-// Disable effects to match false constraints, but if a constraint is true, set
-// the constraint to false to later disable the software effect.
-//
-// This function may modify both |constraints| and |effects|.
-void HarmonizeConstraintsAndEffects(RTCMediaConstraints* constraints,
-                                    int* effects) {
-  if (*effects != media::AudioParameters::NO_EFFECTS) {
-    for (size_t i = 0; i < arraysize(kConstraintEffectMap); ++i) {
-      bool value;
-      size_t is_mandatory = 0;
-      if (!webrtc::FindConstraint(constraints,
-                                  kConstraintEffectMap[i].constraint,
-                                  &value,
-                                  &is_mandatory) || !value) {
-        // If the constraint is false, or does not exist, disable the platform
-        // effect.
-        *effects &= ~kConstraintEffectMap[i].effect;
-        DVLOG(1) << "Disabling platform effect: "
-                 << kConstraintEffectMap[i].effect;
-      } else if (*effects & kConstraintEffectMap[i].effect) {
-        // If the constraint is true, leave the platform effect enabled, and
-        // set the constraint to false to later disable the software effect.
-        if (is_mandatory) {
-          constraints->AddMandatory(kConstraintEffectMap[i].constraint,
-              webrtc::MediaConstraintsInterface::kValueFalse, true);
-        } else {
-          constraints->AddOptional(kConstraintEffectMap[i].constraint,
-              webrtc::MediaConstraintsInterface::kValueFalse, true);
-        }
-        DVLOG(1) << "Disabling constraint: "
-                 << kConstraintEffectMap[i].constraint;
-      }
-    }
-  }
-}
-
 PeerConnectionDependencyFactory::PeerConnectionDependencyFactory(
     P2PSocketDispatcher* p2p_socket_dispatcher)
     : network_manager_(NULL),
       p2p_socket_dispatcher_(p2p_socket_dispatcher),
       signaling_thread_(NULL),
       worker_thread_(NULL),
       chrome_signaling_thread_("Chrome_libJingle_Signaling"),
       chrome_worker_thread_("Chrome_libJingle_WorkerThread") {
   TryScheduleStunProbeTrial();
 }
 
 PeerConnectionDependencyFactory::~PeerConnectionDependencyFactory() {
   DVLOG(1) << "~PeerConnectionDependencyFactory()";
   DCHECK(pc_factory_ == NULL);
 }
 
 blink::WebRTCPeerConnectionHandler*
 PeerConnectionDependencyFactory::CreateRTCPeerConnectionHandler(
     blink::WebRTCPeerConnectionHandlerClient* client) {
   // Save histogram data so we can see how much PeerConnetion is used.
   // The histogram counts the number of calls to the JS API
   // webKitRTCPeerConnection.
   UpdateWebRTCMethodCount(WEBKIT_RTC_PEER_CONNECTION);
 
   return new RTCPeerConnectionHandler(client, this);
 }
 
-bool PeerConnectionDependencyFactory::InitializeMediaStreamAudioSource(
-    int render_frame_id,
-    const blink::WebMediaConstraints& audio_constraints,
-    MediaStreamAudioSource* source_data) {
-  DVLOG(1) << "InitializeMediaStreamAudioSources()";
-
-  // Do additional source initialization if the audio source is a valid
-  // microphone or tab audio.
-  RTCMediaConstraints native_audio_constraints(audio_constraints);
-  MediaAudioConstraints::ApplyFixedAudioConstraints(&native_audio_constraints);
-
-  StreamDeviceInfo device_info = source_data->device_info();
-  RTCMediaConstraints constraints = native_audio_constraints;
-  // May modify both |constraints| and |effects|.
-  HarmonizeConstraintsAndEffects(&constraints,
-                                 &device_info.device.input.effects);
-
-  scoped_refptr<WebRtcAudioCapturer> capturer(CreateAudioCapturer(
-      render_frame_id, device_info, audio_constraints, source_data));
-  if (!capturer.get()) {
-    const std::string log_string =
-        "PCDF::InitializeMediaStreamAudioSource: fails to create capturer";
-    WebRtcLogMessage(log_string);
-    DVLOG(1) << log_string;
-    // TODO(xians): Don't we need to check if source_observer is observing
-    // something? If not, then it looks like we have a leak here.
-    // OTOH, if it _is_ observing something, then the callback might
-    // be called multiple times which is likely also a bug.
-    return false;
-  }
-  source_data->SetAudioCapturer(capturer.get());
-
-  // Creates a LocalAudioSource object which holds audio options.
-  // TODO(xians): The option should apply to the track instead of the source.
-  // TODO(perkj): Move audio constraints parsing to Chrome.
-  // Currently there are a few constraints that are parsed by libjingle and
-  // the state is set to ended if parsing fails.
-  scoped_refptr<webrtc::AudioSourceInterface> rtc_source(
-      CreateLocalAudioSource(&constraints).get());
-  if (rtc_source->state() != webrtc::MediaSourceInterface::kLive) {
-    DLOG(WARNING) << "Failed to create rtc LocalAudioSource.";
-    return false;
-  }
-  source_data->SetLocalAudioSource(rtc_source.get());
-  return true;
-}
-
 WebRtcVideoCapturerAdapter*
 PeerConnectionDependencyFactory::CreateVideoCapturer(
     bool is_screeencast) {
   // We need to make sure the libjingle thread wrappers have been created
   // before we can use an instance of a WebRtcVideoCapturerAdapter. This is
   // since the base class of WebRtcVideoCapturerAdapter is a
   // cricket::VideoCapturer and it uses the libjingle thread wrappers.
   if (!GetPcFactory().get())
     return NULL;
   return new WebRtcVideoCapturerAdapter(is_screeencast);
@@ skipping 296 matching lines @@
 }
 
 scoped_refptr<webrtc::AudioSourceInterface>
 PeerConnectionDependencyFactory::CreateLocalAudioSource(
     const webrtc::MediaConstraintsInterface* constraints) {
   scoped_refptr<webrtc::AudioSourceInterface> source =
       GetPcFactory()->CreateAudioSource(constraints).get();
   return source;
 }
 
-void PeerConnectionDependencyFactory::CreateLocalAudioTrack(
-    const blink::WebMediaStreamTrack& track) {
-  blink::WebMediaStreamSource source = track.source();
-  DCHECK_EQ(source.type(), blink::WebMediaStreamSource::TypeAudio);
-  DCHECK(!source.remote());
-  MediaStreamAudioSource* source_data =
-      static_cast<MediaStreamAudioSource*>(source.extraData());
-
-  scoped_refptr<WebAudioCapturerSource> webaudio_source;
-  if (!source_data) {
-    if (source.requiresAudioConsumer()) {
-      // We're adding a WebAudio MediaStream.
-      // Create a specific capturer for each WebAudio consumer.
-      webaudio_source = CreateWebAudioSource(&source);
-      source_data =
-          static_cast<MediaStreamAudioSource*>(source.extraData());
-    } else {
-      NOTREACHED() << "Local track missing source extra data.";
-      return;
-    }
-  }
-
-  // Creates an adapter to hold all the libjingle objects.
-  scoped_refptr<WebRtcLocalAudioTrackAdapter> adapter(
-      WebRtcLocalAudioTrackAdapter::Create(track.id().utf8(),
-                                           source_data->local_audio_source()));
-  static_cast<webrtc::AudioTrackInterface*>(adapter.get())->set_enabled(
-      track.isEnabled());
-
-  // TODO(xians): Merge |source| to the capturer(). We can't do this today
-  // because only one capturer() is supported while one |source| is created
-  // for each audio track.
-  scoped_ptr<WebRtcLocalAudioTrack> audio_track(new WebRtcLocalAudioTrack(
-      adapter.get(), source_data->GetAudioCapturer(), webaudio_source.get()));
-
-  StartLocalAudioTrack(audio_track.get());
-
-  // Pass the ownership of the native local audio track to the blink track.
-  blink::WebMediaStreamTrack writable_track = track;
-  writable_track.setExtraData(audio_track.release());
-}
-
-void PeerConnectionDependencyFactory::CreateRemoteAudioTrack(
-    const blink::WebMediaStreamTrack& track) {
-  blink::WebMediaStreamSource source = track.source();
-  DCHECK_EQ(source.type(), blink::WebMediaStreamSource::TypeAudio);
-  DCHECK(source.remote());
-  DCHECK(source.extraData());
-
-  blink::WebMediaStreamTrack writable_track = track;
-  writable_track.setExtraData(
-      new MediaStreamRemoteAudioTrack(source, track.isEnabled()));
-}
-
-void PeerConnectionDependencyFactory::StartLocalAudioTrack(
-    WebRtcLocalAudioTrack* audio_track) {
-  // Start the audio track. This will hook the |audio_track| to the capturer
-  // as the sink of the audio, and only start the source of the capturer if
-  // it is the first audio track connecting to the capturer.
-  audio_track->Start();
-}
-
-scoped_refptr<WebAudioCapturerSource>
-PeerConnectionDependencyFactory::CreateWebAudioSource(
-    blink::WebMediaStreamSource* source) {
-  DVLOG(1) << "PeerConnectionDependencyFactory::CreateWebAudioSource()";
-
-  scoped_refptr<WebAudioCapturerSource>
-      webaudio_capturer_source(new WebAudioCapturerSource(*source));
-  MediaStreamAudioSource* source_data = new MediaStreamAudioSource();
-
-  // Use the current default capturer for the WebAudio track so that the
-  // WebAudio track can pass a valid delay value and |need_audio_processing|
-  // flag to PeerConnection.
-  // TODO(xians): Remove this after moving APM to Chrome.
-  if (GetWebRtcAudioDevice()) {
-    source_data->SetAudioCapturer(
-        GetWebRtcAudioDevice()->GetDefaultCapturer());
-  }
-
-  // Create a LocalAudioSource object which holds audio options.
-  // SetLocalAudioSource() affects core audio parts in third_party/Libjingle.
-  source_data->SetLocalAudioSource(CreateLocalAudioSource(NULL).get());
-  source->setExtraData(source_data);
-
-  // Replace the default source with WebAudio as source instead.
-  source->addAudioConsumer(webaudio_capturer_source.get());
-
-  return webaudio_capturer_source;
-}
-
 scoped_refptr<webrtc::VideoTrackInterface>
 PeerConnectionDependencyFactory::CreateLocalVideoTrack(
     const std::string& id,
     webrtc::VideoSourceInterface* source) {
   return GetPcFactory()->CreateVideoTrack(id, source).get();
 }
 
 scoped_refptr<webrtc::VideoTrackInterface>
 PeerConnectionDependencyFactory::CreateLocalVideoTrack(
     const std::string& id, cricket::VideoCapturer* capturer) {
@@ skipping 118 matching lines @@
       // Stopping the thread will wait until all tasks have been
       // processed before returning. We wait for the above task to finish before
       // letting the the function continue to avoid any potential race issues.
       chrome_worker_thread_.Stop();
     } else {
       NOTREACHED() << "Worker thread not running.";
     }
   }
 }
 
-scoped_refptr<WebRtcAudioCapturer>
-PeerConnectionDependencyFactory::CreateAudioCapturer(
-    int render_frame_id,
-    const StreamDeviceInfo& device_info,
-    const blink::WebMediaConstraints& constraints,
-    MediaStreamAudioSource* audio_source) {
-  // TODO(xians): Handle the cases when gUM is called without a proper render
-  // view, for example, by an extension.
-  DCHECK_GE(render_frame_id, 0);
-
-  EnsureWebRtcAudioDeviceImpl();
-  DCHECK(GetWebRtcAudioDevice());
-  return WebRtcAudioCapturer::CreateCapturer(
-      render_frame_id, device_info, constraints, GetWebRtcAudioDevice(),
-      audio_source);
-}
-
 void PeerConnectionDependencyFactory::EnsureInitialized() {
   DCHECK(CalledOnValidThread());
   GetPcFactory();
 }
 
 scoped_refptr<base::SingleThreadTaskRunner>
 PeerConnectionDependencyFactory::GetWebRtcWorkerThread() const {
   DCHECK(CalledOnValidThread());
   return chrome_worker_thread_.IsRunning() ? chrome_worker_thread_.task_runner()
                                            : nullptr;
 }
 
 scoped_refptr<base::SingleThreadTaskRunner>
 PeerConnectionDependencyFactory::GetWebRtcSignalingThread() const {
   DCHECK(CalledOnValidThread());
   return chrome_signaling_thread_.IsRunning()
              ? chrome_signaling_thread_.task_runner()
              : nullptr;
 }
 
 void PeerConnectionDependencyFactory::EnsureWebRtcAudioDeviceImpl() {
   if (audio_device_.get())
     return;
 
   audio_device_ = new WebRtcAudioDeviceImpl();
 }
 
 }  // namespace content