Integrate HTMLMediaElement with Web Audio API's MediaElementAudioSourceNode

content/renderer/media/audio_device.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Low-latency audio rendering unit utilizing audio output stream provided
 // by browser process through IPC.
 //
 // Relationship of classes.
 //
 //  AudioOutputController                AudioDevice
@@ skipping 55 matching lines @@
 #include <vector>
 
 #include "base/basictypes.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/shared_memory.h"
 #include "base/synchronization/lock.h"
 #include "base/threading/simple_thread.h"
 #include "content/common/content_export.h"
 #include "content/renderer/media/audio_message_filter.h"
 #include "media/audio/audio_parameters.h"
+#include "media/filters/audio_renderer_sink.h"
 
 class CONTENT_EXPORT AudioDevice
-    : public AudioMessageFilter::Delegate,
-      public base::DelegateSimpleThread::Delegate,
-      public base::RefCountedThreadSafe<AudioDevice> {
+    : public media::AudioRendererSink,

[henrika (OOO until Aug 14), 2011/12/19 14:27:05]

May I ask how we define Source and Sink in Chrome? What would a similar
interface/filter for capture be called? Is sink is used to denote "receiver/sink
of callbacks"?

[Chris Rogers, 2011/12/19 21:40:35]

"Sink" is just one possible name for the interface.  Some other suggestions are
AudioDestination,
AudioConsumer, AudioEndpoint, AudioReceiver.  Maybe you have some ideas on what
a good name would be?

The idea for the interface is an entity which receives audio by periodically
calling a callback to get
the next N sample-frames of audio.  The Start()/Stop() method will start/stop
the callbacks from happening.
The interface for the callback is called "RenderCallback" and is defined in the
scope of AudioRendererSink (which is the client of that interface).

I would have to study AudioInputDevice in more detail to fully answer your
question about a corresponding interface for capture.  But I think that the most
important feature of the abstraction of an interface would be to implement:
Start();
Stop();
and also to define the CaptureCallback interface for clients of it.

I'm not sure if we will ever need to define this abstraction for
AudioInputDevice, but I think it could be done in a manner consistent with
AudioRendererSink.

[henrika (OOO until Aug 14), 2011/12/20 21:01:01]

The reason I asked is because I sometimes gets confused when I see how sink and
source are used in Chrome. You could perhaps also argue that this interface "is
used as an" audio-feeder to the audio layer, i.e., functions as a source and not
sink.  But I do see your point here. The interface as such does work as a sink
to the actual caller. With a similar line of reasoning for capture, the
interface would define/initiate a data flow from the interface, hence it would
function as a source. Feels OK.

+      public AudioMessageFilter::Delegate,
+      public base::DelegateSimpleThread::Delegate {
  public:
-  class CONTENT_EXPORT RenderCallback {
-   public:
-    virtual void Render(const std::vector<float*>& audio_data,
-                        size_t number_of_frames,
-                        size_t audio_delay_milliseconds) = 0;
-   protected:
-    virtual ~RenderCallback() {}
-  };
-
   // Methods called on main render thread -------------------------------------
 
   // Minimal constructor where Initialize() must be called later.
   AudioDevice();
 
   AudioDevice(size_t buffer_size,
               int channels,
               double sample_rate,
               RenderCallback* callback);
   virtual ~AudioDevice();
 
-  void Initialize(size_t buffer_size,
-                  int channels,
-                  double sample_rate,
-                  AudioParameters::Format latency_format,
-                  RenderCallback* callback);
-  bool IsInitialized();
+  // AudioRendererSink implementation.
 
+  virtual void Initialize(size_t buffer_size,
+                          int channels,
+                          double sample_rate,
+                          AudioParameters::Format latency_format,
+                          RenderCallback* callback) OVERRIDE;
   // Starts audio playback.
-  void Start();
+  virtual void Start() OVERRIDE;
 
   // Stops audio playback.
-  void Stop();
+  virtual void Stop() OVERRIDE;
 
   // Resumes playback if currently paused.
   // TODO(crogers): it should be possible to remove the extra complexity
   // of Play() and Pause() with additional re-factoring work in
   // AudioRendererImpl.
-  void Play();
+  virtual void Play() OVERRIDE;
 
   // Pauses playback.
   // If |flush| is true then any pending audio that is in the pipeline
   // (has not yet reached the hardware) will be discarded.  In this case,
   // when Play() is later called, no previous pending audio will be
   // rendered.
-  void Pause(bool flush);
+  virtual void Pause(bool flush) OVERRIDE;
 
   // Sets the playback volume, with range [0.0, 1.0] inclusive.
   // Returns |true| on success.
-  bool SetVolume(double volume);
+  virtual bool SetVolume(double volume) OVERRIDE;
 
   // Gets the playback volume, with range [0.0, 1.0] inclusive.
   void GetVolume(double* volume);

[henrika (OOO until Aug 14), 2011/12/19 14:27:05]

Any plan to add this to the interface as well?

[Chris Rogers, 2011/12/19 21:40:35]

We easily can.  It might be more symmetric that way.  What do you think?

On 2011/12/19 14:27:05, henrika wrote:

[henrika (OOO until Aug 14), 2011/12/20 21:01:01]

I see now reason why we should make an exception for this method only. 

 
   double sample_rate() const { return sample_rate_; }
   size_t buffer_size() const { return buffer_size_; }
 
   // Methods called on IO thread ----------------------------------------------
   // AudioMessageFilter::Delegate methods, called by AudioMessageFilter.
   virtual void OnRequestPacket(AudioBuffersState buffers_state) OVERRIDE;
   virtual void OnStateChanged(AudioStreamState state) OVERRIDE;
   virtual void OnCreated(base::SharedMemoryHandle handle,
                          uint32 length) OVERRIDE;
@@ skipping 74 matching lines @@
 
   // Protects lifetime of:
   // socket_handle_
   // audio_thread_
   base::Lock lock_;
 
   DISALLOW_COPY_AND_ASSIGN(AudioDevice);
 };
 
 #endif  // CONTENT_RENDERER_MEDIA_AUDIO_DEVICE_H_