Adds MediaStreamSource, MediaStreamAudioSource and MediaStreamVideoCaptureDeviceSource

content/renderer/media/media_stream_dependency_factory.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/media_stream_dependency_factory.h"
 
 #include <vector>
 
 #include "base/command_line.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/synchronization/waitable_event.h"
 #include "content/common/media/media_stream_messages.h"
 #include "content/public/common/content_switches.h"
 #include "content/renderer/media/media_stream_audio_processor_options.h"
-#include "content/renderer/media/media_stream_source_extra_data.h"
+#include "content/renderer/media/media_stream_audio_source.h"
 #include "content/renderer/media/media_stream_track_extra_data.h"
+#include "content/renderer/media/media_stream_video_source.h"
 #include "content/renderer/media/media_stream_video_track.h"
 #include "content/renderer/media/peer_connection_identity_service.h"
 #include "content/renderer/media/rtc_media_constraints.h"
 #include "content/renderer/media/rtc_peer_connection_handler.h"
 #include "content/renderer/media/rtc_video_capturer.h"
 #include "content/renderer/media/rtc_video_decoder_factory.h"
 #include "content/renderer/media/rtc_video_encoder_factory.h"
 #include "content/renderer/media/webaudio_capturer_source.h"
 #include "content/renderer/media/webrtc_audio_device_impl.h"
 #include "content/renderer/media/webrtc_local_audio_track.h"
@@ skipping 85 matching lines @@
  private:
   scoped_refptr<P2PSocketDispatcher> socket_dispatcher_;
   // |network_manager_| and |socket_factory_| are a weak references, owned by
   // MediaStreamDependencyFactory.
   talk_base::NetworkManager* network_manager_;
   talk_base::PacketSocketFactory* socket_factory_;
   // Raw ptr to the WebFrame that created the P2PPortAllocatorFactory.
   blink::WebFrame* web_frame_;
 };
 
-// SourceStateObserver is a help class used for observing the startup state
-// transition of webrtc media sources such as a camera or microphone.
-// An instance of the object deletes itself after use.
-// Usage:
-// 1. Create an instance of the object with the blink::WebMediaStream
-//    the observed sources belongs to a callback.
-// 2. Add the sources to the observer using AddSource.
-// 3. Call StartObserving()
-// 4. The callback will be triggered when all sources have transitioned from
-//    webrtc::MediaSourceInterface::kInitializing.
-class SourceStateObserver : public webrtc::ObserverInterface,
-                            public base::NonThreadSafe {
- public:
-  SourceStateObserver(
-      blink::WebMediaStream* web_stream,
-      const MediaStreamDependencyFactory::MediaSourcesCreatedCallback& callback)
-     : web_stream_(web_stream),
-       ready_callback_(callback),
-       live_(true) {
-  }
-
-  void AddSource(webrtc::MediaSourceInterface* source) {
-    DCHECK(CalledOnValidThread());
-    switch (source->state()) {
-      case webrtc::MediaSourceInterface::kInitializing:
-        sources_.push_back(source);
-        source->RegisterObserver(this);
-        break;
-      case webrtc::MediaSourceInterface::kLive:
-        // The source is already live so we don't need to wait for it.
-        break;
-      case webrtc::MediaSourceInterface::kEnded:
-        // The source have already failed.
-        live_ = false;
-        break;
-      default:
-        NOTREACHED();
-    }
-  }
-
-  void StartObservering() {
-    DCHECK(CalledOnValidThread());
-    CheckIfSourcesAreLive();
-  }
-
-  virtual void OnChanged() OVERRIDE {
-    DCHECK(CalledOnValidThread());
-    CheckIfSourcesAreLive();
-  }
-
- private:
-  void CheckIfSourcesAreLive() {
-    ObservedSources::iterator it = sources_.begin();
-    while (it != sources_.end()) {
-      if ((*it)->state() != webrtc::MediaSourceInterface::kInitializing) {
-        live_ &=  (*it)->state() == webrtc::MediaSourceInterface::kLive;
-        (*it)->UnregisterObserver(this);
-        it = sources_.erase(it);
-      } else {
-        ++it;
-      }
-    }
-    if (sources_.empty()) {
-      ready_callback_.Run(web_stream_, live_);
-      delete this;
-    }
-  }
-
-  blink::WebMediaStream* web_stream_;
-  MediaStreamDependencyFactory::MediaSourcesCreatedCallback ready_callback_;
-  bool live_;
-  typedef std::vector<scoped_refptr<webrtc::MediaSourceInterface> >
-      ObservedSources;
-  ObservedSources sources_;
-};
-
 MediaStreamDependencyFactory::MediaStreamDependencyFactory(
     P2PSocketDispatcher* p2p_socket_dispatcher)
     : network_manager_(NULL),
       p2p_socket_dispatcher_(p2p_socket_dispatcher),
       signaling_thread_(NULL),
       worker_thread_(NULL),
       chrome_worker_thread_("Chrome_libJingle_WorkerThread"),
       aec_dump_file_(base::kInvalidPlatformFileValue) {
 }
 
@@ skipping 10 matching lines @@
   // The histogram counts the number of calls to the JS API
   // webKitRTCPeerConnection.
   UpdateWebRTCMethodCount(WEBKIT_RTC_PEER_CONNECTION);
 
   if (!EnsurePeerConnectionFactory())
     return NULL;
 
   return new RTCPeerConnectionHandler(client, this);
 }
 
-void MediaStreamDependencyFactory::CreateNativeMediaSources(
+bool MediaStreamDependencyFactory::InitializeMediaStreamAudioSource(
     int render_view_id,
     const blink::WebMediaConstraints& audio_constraints,
-    const blink::WebMediaConstraints& video_constraints,
-    blink::WebMediaStream* web_stream,
-    const MediaSourcesCreatedCallback& sources_created) {
-  DVLOG(1) << "MediaStreamDependencyFactory::CreateNativeMediaSources()";
-  if (!EnsurePeerConnectionFactory()) {
-    sources_created.Run(web_stream, false);
-    return;
-  }
-
-  // |source_observer| clean up itself when it has completed
-  // source_observer->StartObservering.
-  SourceStateObserver* source_observer =
-      new SourceStateObserver(web_stream, sources_created);
-
-  // Create local video sources.
-  RTCMediaConstraints native_video_constraints(video_constraints);
-  blink::WebVector<blink::WebMediaStreamTrack> video_tracks;
-  web_stream->videoTracks(video_tracks);
-  for (size_t i = 0; i < video_tracks.size(); ++i) {
-    const blink::WebMediaStreamSource& source = video_tracks[i].source();
-    MediaStreamSourceExtraData* source_data =
-        static_cast<MediaStreamSourceExtraData*>(source.extraData());
-
-    // Check if the source has already been created. This happens when the same
-    // source is used in multiple MediaStreams as a result of calling
-    // getUserMedia.
-    if (source_data->video_source())
-      continue;
-
-    const bool is_screencast =
-        source_data->device_info().device.type == MEDIA_TAB_VIDEO_CAPTURE ||
-        source_data->device_info().device.type == MEDIA_DESKTOP_VIDEO_CAPTURE;
-    source_data->SetVideoSource(
-        CreateLocalVideoSource(source_data->device_info().session_id,
-                               is_screencast,
-                               &native_video_constraints).get());
-    source_observer->AddSource(source_data->video_source());
-  }
+    const blink::WebMediaStreamSource& audio_source) {
+  DVLOG(1) << "InitializeMediaStreamAudioSources()";
 
   // Do additional source initialization if the audio source is a valid
   // microphone or tab audio.
   RTCMediaConstraints native_audio_constraints(audio_constraints);
   ApplyFixedAudioConstraints(&native_audio_constraints);
-  blink::WebVector<blink::WebMediaStreamTrack> audio_tracks;
-  web_stream->audioTracks(audio_tracks);
-  for (size_t i = 0; i < audio_tracks.size(); ++i) {
-    const blink::WebMediaStreamSource& source = audio_tracks[i].source();
-    MediaStreamSourceExtraData* source_data =
-        static_cast<MediaStreamSourceExtraData*>(source.extraData());
 
-    // Check if the source has already been created. This happens when the same
-    // source is used in multiple MediaStreams as a result of calling
-    // getUserMedia.
-    if (source_data->local_audio_source())
-      continue;
+  MediaStreamAudioSource* source_data =
+      static_cast<MediaStreamAudioSource*>(audio_source.extraData());
 
-    // TODO(xians): Create a new capturer for difference microphones when we
-    // support multiple microphones. See issue crbug/262117 .
-    StreamDeviceInfo device_info = source_data->device_info();
-    RTCMediaConstraints constraints = native_audio_constraints;
+  // TODO(xians): Create a new capturer for difference microphones when we
+  // support multiple microphones. See issue crbug/262117 .
+  StreamDeviceInfo device_info = source_data->device_info();
+  RTCMediaConstraints constraints = native_audio_constraints;
 
-    // If any platform effects are available, check them against the
-    // constraints. Disable effects to match false constraints, but if a
-    // constraint is true, set the constraint to false to later disable the
-    // software effect.
-    int effects = device_info.device.input.effects;
-    if (effects != media::AudioParameters::NO_EFFECTS) {
-      for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConstraintEffectMap); ++i) {
-        bool value;
-        if (!webrtc::FindConstraint(&constraints,
-                kConstraintEffectMap[i].constraint, &value, NULL) || !value) {
-          // If the constraint is false, or does not exist, disable the platform
-          // effect.
-          effects &= ~kConstraintEffectMap[i].effect;
-          DVLOG(1) << "Disabling constraint: "
-                   << kConstraintEffectMap[i].constraint;
-        } else if (effects & kConstraintEffectMap[i].effect) {
-          // If the constraint is true, leave the platform effect enabled, and
-          // set the constraint to false to later disable the software effect.
-          constraints.AddMandatory(kConstraintEffectMap[i].constraint,
-              webrtc::MediaConstraintsInterface::kValueFalse, true);
-          DVLOG(1) << "Disabling platform effect: "
-                   << kConstraintEffectMap[i].constraint;
-        }
+  // If any platform effects are available, check them against the
+  // constraints. Disable effects to match false constraints, but if a
+  // constraint is true, set the constraint to false to later disable the
+  // software effect.
+  int effects = device_info.device.input.effects;
+  if (effects != media::AudioParameters::NO_EFFECTS) {
+    for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConstraintEffectMap); ++i) {
+      bool value;
+      if (!webrtc::FindConstraint(&constraints,
+                                  kConstraintEffectMap[i].constraint, &value,
+                                  NULL) || !value) {
+        // If the constraint is false, or does not exist, disable the platform
+        // effect.
+        effects &= ~kConstraintEffectMap[i].effect;
+        DVLOG(1) << "Disabling constraint: "
+            << kConstraintEffectMap[i].constraint;
+      } else if (effects & kConstraintEffectMap[i].effect) {
+        // If the constraint is true, leave the platform effect enabled, and
+        // set the constraint to false to later disable the software effect.
+        constraints.AddMandatory(kConstraintEffectMap[i].constraint,
+                                 webrtc::MediaConstraintsInterface::kValueFalse,
+                                 true);
+        DVLOG(1) << "Disabling platform effect: "
+            << kConstraintEffectMap[i].constraint;
       }
-      device_info.device.input.effects = effects;
     }
-
-    scoped_refptr<WebRtcAudioCapturer> capturer(
-        CreateAudioCapturer(render_view_id, device_info, audio_constraints));
-    if (!capturer.get()) {
-      DLOG(WARNING) << "Failed to create the capturer for device "
-                    << device_info.device.id;
-      sources_created.Run(web_stream, false);
-      // TODO(xians): Don't we need to check if source_observer is observing
-      // something? If not, then it looks like we have a leak here.
-      // OTOH, if it _is_ observing something, then the callback might
-      // be called multiple times which is likely also a bug.
-      return;
-    }
-    source_data->SetAudioCapturer(capturer);
-
-    // Creates a LocalAudioSource object which holds audio options.
-    // TODO(xians): The option should apply to the track instead of the source.
-    source_data->SetLocalAudioSource(
-        CreateLocalAudioSource(&constraints).get());
-    source_observer->AddSource(source_data->local_audio_source());
+    device_info.device.input.effects = effects;
   }
 
-  source_observer->StartObservering();
+  scoped_refptr<WebRtcAudioCapturer> capturer(
+      CreateAudioCapturer(render_view_id, device_info, audio_constraints));
+  if (!capturer.get()) {
+    DLOG(WARNING) << "Failed to create the capturer for device "
+        << device_info.device.id;
+    // TODO(xians): Don't we need to check if source_observer is observing
+    // something? If not, then it looks like we have a leak here.
+    // OTOH, if it _is_ observing something, then the callback might
+    // be called multiple times which is likely also a bug.
+    return false;
+  }
+  source_data->SetAudioCapturer(capturer);
+
+  // Creates a LocalAudioSource object which holds audio options.
+  // TODO(xians): The option should apply to the track instead of the source.
+  source_data->SetLocalAudioSource(
+      CreateLocalAudioSource(&constraints).get());
+
+  return true;
+}
+
+cricket::VideoCapturer* MediaStreamDependencyFactory::CreateVideoCapturer(
+    const StreamDeviceInfo& info) {
+  bool is_screeencast =
+      info.device.type == MEDIA_TAB_VIDEO_CAPTURE ||
+      info.device.type == MEDIA_DESKTOP_VIDEO_CAPTURE;
+  return new RtcVideoCapturer(info.session_id, is_screeencast);
 }
 
 void MediaStreamDependencyFactory::CreateNativeLocalMediaStream(
     blink::WebMediaStream* web_stream) {
   DVLOG(1) << "MediaStreamDependencyFactory::CreateNativeLocalMediaStream()";
   if (!EnsurePeerConnectionFactory()) {
     DVLOG(1) << "EnsurePeerConnectionFactory() failed!";
     return;
   }
 
@@ skipping 27 matching lines @@
   MediaStreamExtraData* extra_data =
      static_cast<MediaStreamExtraData*>(web_stream->extraData());
   extra_data->SetLocalStreamStopCallback(stream_stop);
 }
 
 scoped_refptr<webrtc::AudioTrackInterface>
 MediaStreamDependencyFactory::CreateNativeAudioMediaStreamTrack(
     const blink::WebMediaStreamTrack& track) {
   blink::WebMediaStreamSource source = track.source();
   DCHECK_EQ(source.type(), blink::WebMediaStreamSource::TypeAudio);
-  MediaStreamSourceExtraData* source_data =
-      static_cast<MediaStreamSourceExtraData*>(source.extraData());
+  MediaStreamAudioSource* source_data =
+      static_cast<MediaStreamAudioSource*>(source.extraData());
 
   // In the future the constraints will belong to the track itself, but
   // right now they're on the source, so we fetch them from there.
   RTCMediaConstraints track_constraints(source.constraints());
 
   // Apply default audio constraints that enable echo cancellation,
   // automatic gain control, noise suppression and high-pass filter.
   ApplyFixedAudioConstraints(&track_constraints);
 
   scoped_refptr<WebAudioCapturerSource> webaudio_source;
   if (!source_data) {
     if (source.requiresAudioConsumer()) {
       // We're adding a WebAudio MediaStream.
       // Create a specific capturer for each WebAudio consumer.
       webaudio_source = CreateWebAudioSource(&source, track_constraints);
       source_data =
-          static_cast<MediaStreamSourceExtraData*>(source.extraData());
+          static_cast<MediaStreamAudioSource*>(source.extraData());
     } else {
       // TODO(perkj): Implement support for sources from
       // remote MediaStreams.
       NOTIMPLEMENTED();
       return NULL;
     }
   }
 
   scoped_refptr<webrtc::AudioTrackInterface> audio_track(
       CreateLocalAudioTrack(track.id().utf8(),
                             source_data->GetAudioCapturer(),
                             webaudio_source.get(),
                             source_data->local_audio_source()));
   AddNativeTrackToBlinkTrack(audio_track.get(), track, true);
 
   audio_track->set_enabled(track.isEnabled());
 
   // Pass the pointer of the source provider to the blink audio track.
   blink::WebMediaStreamTrack writable_track = track;
   writable_track.setSourceProvider(static_cast<WebRtcLocalAudioTrack*>(
       audio_track.get())->audio_source_provider());
 
   return audio_track;
 }
 
 scoped_refptr<webrtc::VideoTrackInterface>
 MediaStreamDependencyFactory::CreateNativeVideoMediaStreamTrack(
     const blink::WebMediaStreamTrack& track) {
+  DCHECK(track.extraData() == NULL);
   blink::WebMediaStreamSource source = track.source();
   DCHECK_EQ(source.type(), blink::WebMediaStreamSource::TypeVideo);
-  MediaStreamSourceExtraData* source_data =
-      static_cast<MediaStreamSourceExtraData*>(source.extraData());
+
+  MediaStreamVideoSource* source_data =
+      static_cast<MediaStreamVideoSource*>(source.extraData());
 
   if (!source_data) {
     // TODO(perkj): Implement support for sources from
     // remote MediaStreams.
     NOTIMPLEMENTED();
     return NULL;
   }
 
-  std::string track_id = base::UTF16ToUTF8(track.id());
-  scoped_refptr<webrtc::VideoTrackInterface> video_track(
-      CreateLocalVideoTrack(track_id, source_data->video_source()));
-  AddNativeTrackToBlinkTrack(video_track.get(), track, true);
+  // Create native track from the source.
+  scoped_refptr<webrtc::VideoTrackInterface> webrtc_track =
+      CreateLocalVideoTrack(track.id().utf8(), source_data->GetAdapter());
 
-  video_track->set_enabled(track.isEnabled());
+  AddNativeTrackToBlinkTrack(webrtc_track, track, true);

[Ronghua Wu (Left Chromium), 2014/01/22 01:44:04]

nit: bool local_track = true;

[perkj_chrome, 2014/01/22 16:56:37]

Done.

 
-  return video_track;
+  webrtc_track->set_enabled(track.isEnabled());
+
+  return webrtc_track;
 }
 
 void MediaStreamDependencyFactory::CreateNativeMediaStreamTrack(
     const blink::WebMediaStreamTrack& track) {
   DCHECK(!track.isNull() && !track.extraData());
   DCHECK(!track.source().isNull());
 
   switch (track.source().type()) {
     case blink::WebMediaStreamSource::TypeAudio:
       CreateNativeAudioMediaStreamTrack(track);
@@ skipping 90 matching lines @@
       return native_stream->RemoveTrack(
           native_stream->FindVideoTrack(track_id));
   }
   return false;
 }
 
 scoped_refptr<webrtc::VideoSourceInterface>
     MediaStreamDependencyFactory::CreateVideoSource(
         cricket::VideoCapturer* capturer,
         const webrtc::MediaConstraintsInterface* constraints) {
+  if (!EnsurePeerConnectionFactory()) {

[Ronghua Wu (Left Chromium), 2014/01/22 01:44:04]

It's unclear to me what is the rule to call or not call
EnsurePeerConnectionFactory in the beginning of those functions.

We can create a factory getter function and put EnsurePeerConnectionFactory
there once for everyone.

[perkj_chrome, 2014/01/22 16:56:37]

good idea

+    return NULL;
+  }
   scoped_refptr<webrtc::VideoSourceInterface> source =
       pc_factory_->CreateVideoSource(capturer, constraints).get();
   return source;
 }
 
 bool MediaStreamDependencyFactory::CreatePeerConnectionFactory() {
   DCHECK(!pc_factory_.get());
   DCHECK(!audio_device_.get());
   DVLOG(1) << "MediaStreamDependencyFactory::CreatePeerConnectionFactory()";
 
@@ skipping 84 matching lines @@
 
 scoped_refptr<webrtc::MediaStreamInterface>
 MediaStreamDependencyFactory::CreateLocalMediaStream(
     const std::string& label) {
   return pc_factory_->CreateLocalMediaStream(label).get();
 }
 
 scoped_refptr<webrtc::AudioSourceInterface>
 MediaStreamDependencyFactory::CreateLocalAudioSource(
     const webrtc::MediaConstraintsInterface* constraints) {
+  if (!EnsurePeerConnectionFactory())
+    return NULL;
+
   scoped_refptr<webrtc::AudioSourceInterface> source =
       pc_factory_->CreateAudioSource(constraints).get();
   return source;
 }
 
-scoped_refptr<webrtc::VideoSourceInterface>
-MediaStreamDependencyFactory::CreateLocalVideoSource(
-    int video_session_id,
-    bool is_screencast,
-    const webrtc::MediaConstraintsInterface* constraints) {
-  RtcVideoCapturer* capturer = new RtcVideoCapturer(
-      video_session_id, is_screencast);
-
-  // The video source takes ownership of |capturer|.
-  scoped_refptr<webrtc::VideoSourceInterface> source =
-      CreateVideoSource(capturer, constraints);
-  return source;
-}
-
 scoped_refptr<WebAudioCapturerSource>
 MediaStreamDependencyFactory::CreateWebAudioSource(
     blink::WebMediaStreamSource* source,
     const RTCMediaConstraints& constraints) {
   DVLOG(1) << "MediaStreamDependencyFactory::CreateWebAudioSource()";
   DCHECK(GetWebRtcAudioDevice());
 
   scoped_refptr<WebAudioCapturerSource>
       webaudio_capturer_source(new WebAudioCapturerSource());
-  MediaStreamSourceExtraData* source_data = new MediaStreamSourceExtraData();
+  MediaStreamAudioSource* source_data = new MediaStreamAudioSource();
 
   // Create a LocalAudioSource object which holds audio options.
   // SetLocalAudioSource() affects core audio parts in third_party/Libjingle.
   source_data->SetLocalAudioSource(CreateLocalAudioSource(&constraints).get());
   source->setExtraData(source_data);
 
   // Replace the default source with WebAudio as source instead.
   source->addAudioConsumer(webaudio_capturer_source.get());
 
   return webaudio_capturer_source;
@@ skipping 254 matching lines @@
   if (!aec_dump_file_stream) {
     VLOG(1) << "Could not open AEC dump file.";
     base::ClosePlatformFile(aec_dump_file);
   } else {
     // |pc_factory_| takes ownership of |aec_dump_file_stream|.
     pc_factory_->StartAecDump(aec_dump_file_stream);
   }
 }
 
 }  // namespace content