Implement suspend() and resume() for OfflineAudioContext

third_party/WebKit/Source/modules/webaudio/OfflineAudioDestinationNode.cpp

 /*
  * Copyright (C) 2011, Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1.  Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2.  Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 10 matching lines @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "config.h"
 #if ENABLE(WEB_AUDIO)
 #include "modules/webaudio/OfflineAudioDestinationNode.h"
 
 #include "core/dom/CrossThreadTask.h"
 #include "modules/webaudio/AbstractAudioContext.h"
+#include "modules/webaudio/AudioNodeInput.h"
+#include "modules/webaudio/AudioNodeOutput.h"
+#include "modules/webaudio/OfflineAudioContext.h"
 #include "platform/Task.h"
 #include "platform/audio/AudioBus.h"
+#include "platform/audio/DenormalDisabler.h"
 #include "platform/audio/HRTFDatabaseLoader.h"
 #include "public/platform/Platform.h"
 #include <algorithm>
 
 namespace blink {
 
 const size_t renderQuantumSize = 128;
 
 OfflineAudioDestinationHandler::OfflineAudioDestinationHandler(AudioNode& node, AudioBuffer* renderTarget)
     : AudioDestinationHandler(node, renderTarget->sampleRate())
     , m_renderTarget(renderTarget)
-    , m_startedRendering(false)
+    , m_renderThread(adoptPtr(Platform::current()->createThread("offline audio renderer")))
+    , m_framesProcessed(0)
+    , m_framesToProcess(0)
+    , m_isRenderingStarted(false)
+    , m_shouldSuspend(false)
 {
     m_renderBus = AudioBus::create(renderTarget->numberOfChannels(), renderQuantumSize);

[Raymond Toy, 2015/10/21 18:22:46]

What if AudioBus::create fails to allocate space?

[hongchan, 2015/10/22 18:23:49]

That would be an exception.

Also without a valid m_renderBus, the rendering cannot be started. Still we are
better off throwing an exception for this case, and I think that should be
another CL.

 }
 
 PassRefPtr<OfflineAudioDestinationHandler> OfflineAudioDestinationHandler::create(AudioNode& node, AudioBuffer* renderTarget)
 {
     return adoptRef(new OfflineAudioDestinationHandler(node, renderTarget));
 }
 
 OfflineAudioDestinationHandler::~OfflineAudioDestinationHandler()
 {
     ASSERT(!isInitialized());
@@ skipping 17 matching lines @@
 {
     if (!isInitialized())
         return;
 
     if (m_renderThread)
         m_renderThread.clear();
 
     AudioHandler::uninitialize();
 }
 
+OfflineAudioContext* OfflineAudioDestinationHandler::context() const
+{
+    return static_cast<OfflineAudioContext*>(m_context);
+}
+
 void OfflineAudioDestinationHandler::startRendering()
 {
     ASSERT(isMainThread());
     ASSERT(m_renderTarget);
+    ASSERT(m_renderThread);

[Raymond Toy, 2015/10/21 18:22:46]

I would reverse these so that ASSERT(m_renderTarget) is closer to the if.

[hongchan, 2015/10/22 18:23:49]

Done.

+
     if (!m_renderTarget)
         return;
 
-    if (!m_startedRendering) {
-        m_startedRendering = true;
-        m_renderThread = adoptPtr(Platform::current()->createThread("Offline Audio Renderer"));
-        m_renderThread->taskRunner()->postTask(BLINK_FROM_HERE, new Task(threadSafeBind(&OfflineAudioDestinationHandler::offlineRender, PassRefPtr<OfflineAudioDestinationHandler>(this))));
+    // Rendering was not started. Starting now.
+    if (!m_isRenderingStarted) {
+        m_renderThread->taskRunner()->postTask(BLINK_FROM_HERE,
+            new Task(threadSafeBind(&OfflineAudioDestinationHandler::startOfflineRendering, this)));
+        m_isRenderingStarted = true;

[Raymond Toy, 2015/10/21 18:22:46]

Does it matter if m_isRenderingStarted is set to true after posting the task? 
Does startOfflineRendering every check m_isRenderingStarted?

[hongchan, 2015/10/22 18:23:49]

m_isRenderingStart can only be read/modified by startRendering() - which is
invoked by OAC.startRendering(). The rendering loop does not check
m_isRenderingStarted, so I think this is safe.

However, I can move it up before posting a task.

+        return;
     }
+
+    // Rendering is already started, which implicitly means we resume the
+    // rendering by calling |doOfflineRendering| on the render thread.
+    m_renderThread->taskRunner()->postTask(BLINK_FROM_HERE,
+        threadSafeBind(&OfflineAudioDestinationHandler::doOfflineRendering, this));
 }
 
 void OfflineAudioDestinationHandler::stopRendering()
 {
+    // OfflineAudioContext CANNOT BE stopped by JavaScript.
     ASSERT_NOT_REACHED();
 }
 
-void OfflineAudioDestinationHandler::offlineRender()
+WebThread* OfflineAudioDestinationHandler::offlineRenderThread()
 {
-    offlineRenderInternal();
-    context()->handlePostRenderTasks();
+    ASSERT(m_renderThread);
+
+    return m_renderThread.get();
 }
 
-void OfflineAudioDestinationHandler::offlineRenderInternal()
+size_t OfflineAudioDestinationHandler::renderQuantumFrames() const

[Raymond Toy, 2015/10/21 18:22:45]

Should this be inline?  It's so small.  Or maybe make renderQuantumSize a class
static constant?

[hongchan, 2015/10/22 18:23:49]

Done.

+{
+    return renderQuantumSize;
+}
+
+void OfflineAudioDestinationHandler::startOfflineRendering()
 {
     ASSERT(!isMainThread());
+
     ASSERT(m_renderBus);
     if (!m_renderBus)
         return;
 
     bool isAudioContextInitialized = context()->isDestinationInitialized();
     ASSERT(isAudioContextInitialized);
     if (!isAudioContextInitialized)
         return;
 
     bool channelsMatch = m_renderBus->numberOfChannels() == m_renderTarget->numberOfChannels();
     ASSERT(channelsMatch);
     if (!channelsMatch)
         return;
 
     bool isRenderBusAllocated = m_renderBus->length() >= renderQuantumSize;
     ASSERT(isRenderBusAllocated);
     if (!isRenderBusAllocated)
         return;
 
     // Break up the render target into smaller "render quantize" sized pieces.
     // Render until we're finished.
-    size_t framesToProcess = m_renderTarget->length();
+    m_framesToProcess = m_renderTarget->length();

[Raymond Toy, 2015/10/21 18:22:46]

Shouldn't this be set when the offline context is created where the
m_renderTarget is created with the actual output buffer?

And I don't understand how this comment is relevant to this line of code.

[hongchan, 2015/10/22 18:23:49]

Done, and I mistakenly kept the old comment. I will remove it.

+
+    // Start rendering.
+    doOfflineRendering();
+}
+
+void OfflineAudioDestinationHandler::doOfflineRendering()
+{
+    ASSERT(!isMainThread());
+
     unsigned numberOfChannels = m_renderTarget->numberOfChannels();
 
-    unsigned n = 0;
-    while (framesToProcess > 0) {
-        // Render one render quantum.
-        render(0, m_renderBus.get(), renderQuantumSize);
+    // Reset the suspend flag.
+    m_shouldSuspend = false;
 
-        size_t framesAvailableToCopy = std::min(framesToProcess, renderQuantumSize);
+    // If there is more to process and there is no suspension at the moment,
+    // do continue to render quanta. Then calling OfflineAudioContext.resume() will pick up
+    // the render loop again from where it was suspended.
+    while (m_framesToProcess > 0 && !m_shouldSuspend) {
+
+        // Render one render quantum. Note that this includes pre/post render
+        // tasks from the online audio context. Note that this method will
+        // change |m_shouldSuspend| internally according to the scheduled

[Raymond Toy, 2015/10/21 18:22:46]

This comment about changing m_shouldSuspend internally is now wrong, right?

[hongchan, 2015/10/22 18:23:49]

Oops. Done.

+        // suspends.
+        m_shouldSuspend = checkSuspendsAndRender(0, m_renderBus.get(), renderQuantumSize);
+
+        if (m_shouldSuspend) {
+            suspendOfflineRendering();
+            return;
+        }
+
+        size_t framesAvailableToCopy = std::min(m_framesToProcess, renderQuantumSize);
 
         for (unsigned channelIndex = 0; channelIndex < numberOfChannels; ++channelIndex) {
             const float* source = m_renderBus->channel(channelIndex)->data();
             float* destination = m_renderTarget->getChannelData(channelIndex)->data();
-            memcpy(destination + n, source, sizeof(float) * framesAvailableToCopy);
+            memcpy(destination + m_framesProcessed, source, sizeof(float) * framesAvailableToCopy);
         }
 
-        n += framesAvailableToCopy;
-        framesToProcess -= framesAvailableToCopy;
+        m_framesProcessed += framesAvailableToCopy;
+
+        ASSERT(m_framesToProcess >= framesAvailableToCopy);
+        m_framesToProcess -= framesAvailableToCopy;
     }
 
-    // Our work is done. Let the AbstractAudioContext know.
-    if (context()->executionContext())
-        context()->executionContext()->postTask(BLINK_FROM_HERE, createCrossThreadTask(&OfflineAudioDestinationHandler::notifyComplete, PassRefPtr<OfflineAudioDestinationHandler>(this)));
+    // Finish up the rendering loop if there is no more to process.
+    if (m_framesToProcess == 0) {

[Raymond Toy, 2015/10/21 18:22:46]

I hate the style, but you're not supposed to compare against 0.  Just if
(!m_framesToProcess)

[hongchan, 2015/10/22 18:23:49]

Done.

+        ASSERT(m_framesToProcess == 0);

[Raymond Toy, 2015/10/21 18:22:46]

How can this assert ever fail?  Delete it.

[hongchan, 2015/10/22 18:23:49]

This was suggested by haraken@ before, but I think I can remove this.

+        finishOfflineRendering();
+    }
+}
+
+void OfflineAudioDestinationHandler::suspendOfflineRendering()
+{
+    ASSERT(!isMainThread());
+
+    // The actual rendering has been suspended. Notify the context.
+    if (context()->executionContext()) {
+        context()->executionContext()->postTask(BLINK_FROM_HERE,
+            createCrossThreadTask(&OfflineAudioDestinationHandler::notifySuspend, this));
+    }
+}
+
+void OfflineAudioDestinationHandler::finishOfflineRendering()
+{
+    ASSERT(!isMainThread());
+
+    // The actual rendering has been completed. Notify the context.
+    if (context()->executionContext()) {
+        context()->executionContext()->postTask(BLINK_FROM_HERE,
+            createCrossThreadTask(&OfflineAudioDestinationHandler::notifyComplete, this));
+    }
+}
+
+void OfflineAudioDestinationHandler::notifySuspend()
+{
+    if (context())
+        context()->resolveSuspendOnMainThread(context()->currentSampleFrame());
 }
 
 void OfflineAudioDestinationHandler::notifyComplete()
 {
-    // The AbstractAudioContext might be gone.
+    // The OfflineAudioContext might be gone.
     if (context())
         context()->fireCompletionEvent();
 }
 
+bool OfflineAudioDestinationHandler::checkSuspendsAndRender(AudioBus* sourceBus, AudioBus* destinationBus, size_t numberOfFrames)
+{
+    // We don't want denormals slowing down any of the audio processing
+    // since they can very seriously hurt performance.
+    // This will take care of all AudioNodes because they all process within this scope.
+    DenormalDisabler denormalDisabler;
+
+    context()->deferredTaskHandler().setAudioThread(currentThread());
+
+    if (!context()->isDestinationInitialized()) {

[Raymond Toy, 2015/10/21 18:22:45]

For an offline context, is it actually possible for the destination not to be
initialized here?

[hongchan, 2015/10/22 18:23:49]

This method is just a ported version of render() method from
AudioDestinatioNode. So I try not to change the code if the part is not relevant
to suspend/resume.

However, offline audio context does not have to wait for the audio device, we
might be safe without this check. I will investigate this further.

+        destinationBus->zero();
+        return false;
+    }
+
+    // Take care pre-render tasks at the beginning of each render quantum. This
+    // will change |m_shouldSuspend| flag if there is a suspend scheduled at the

[Raymond Toy, 2015/10/21 18:22:46]

Comment is wrong? m_shouldSuspend isn't modified.

[hongchan, 2015/10/22 18:23:49]

Oops. Done.

+    // current frame.
+    if (context()->handlePreOfflineRenderTasks())
+        return true;
+
+    // Prepare the local audio input provider for this render quantum.
+    if (sourceBus)
+        m_localAudioInputProvider.set(sourceBus);
+
+    ASSERT(numberOfInputs() >= 1);
+    if (numberOfInputs() < 1) {
+        destinationBus->zero();
+        return false;
+    }
+    // This will cause the node(s) connected to us to process, which in turn will pull on their input(s),
+    // all the way backwards through the rendering graph.
+    AudioBus* renderedBus = input(0).pull(destinationBus, numberOfFrames);
+
+    if (!renderedBus) {
+        destinationBus->zero();
+    } else if (renderedBus != destinationBus) {
+        // in-place processing was not possible - so copy
+        destinationBus->copyFrom(*renderedBus);
+    }
+
+    // Process nodes which need a little extra help because they are not connected to anything, but still need to process.
+    context()->deferredTaskHandler().processAutomaticPullNodes(numberOfFrames);
+
+    // Let the context take care of any business at the end of each render quantum.
+    context()->handlePostOfflineRenderTasks();
+
+    // Advance current sample-frame.
+    size_t newSampleFrame = m_currentSampleFrame + numberOfFrames;
+    releaseStore(&m_currentSampleFrame, newSampleFrame);
+
+    return false;
+}
+
 // ----------------------------------------------------------------
 
 OfflineAudioDestinationNode::OfflineAudioDestinationNode(AbstractAudioContext& context, AudioBuffer* renderTarget)
     : AudioDestinationNode(context)
 {
     setHandler(OfflineAudioDestinationHandler::create(*this, renderTarget));
 }
 
 OfflineAudioDestinationNode* OfflineAudioDestinationNode::create(AbstractAudioContext* context, AudioBuffer* renderTarget)
 {
     return new OfflineAudioDestinationNode(*context, renderTarget);
 }
 
 } // namespace blink
 
 #endif // ENABLE(WEB_AUDIO)