Add an initial revision of an audio server.

services/media/audio/platform/generic/standard_output_base.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <limits>
+
+#include "base/logging.h"
+#include "services/media/audio/audio_track_impl.h"
+#include "services/media/audio/audio_track_to_output_link.h"
+#include "services/media/audio/platform/generic/mixer.h"
+#include "services/media/audio/platform/generic/standard_output_base.h"
+
+namespace mojo {
+namespace media {
+namespace audio {
+
+static constexpr LocalDuration MAX_TRIM_PERIOD = local_time::from_msec(10);
+constexpr uint32_t StandardOutputBase::MixJob::INVALID_GENERATION;
+
+StandardOutputBase::TrackBookkeeping::TrackBookkeeping() {}
+StandardOutputBase::TrackBookkeeping::~TrackBookkeeping() {}
+
+StandardOutputBase::StandardOutputBase(AudioOutputManager* manager)
+  : AudioOutput(manager) {
+  setup_mix_ =
+    [this] (const AudioTrackImplPtr& track, TrackBookkeeping* info) -> bool {
+      return SetupMix(track, info);
+    };
+
+  process_mix_ =
+    [this] (const AudioTrackImplPtr& track,
+            TrackBookkeeping* info,
+            const AudioPipe::AudioPacketRefPtr& pkt_ref) -> bool {
+      return ProcessMix(track, info, pkt_ref);
+    };
+
+  setup_trim_ =
+    [this] (const AudioTrackImplPtr& track, TrackBookkeeping* info) -> bool {
+      return SetupTrim(track, info);
+    };
+
+  process_trim_ =
+    [this] (const AudioTrackImplPtr& track,
+            TrackBookkeeping* info,
+            const AudioPipe::AudioPacketRefPtr& pkt_ref) -> bool {
+      return ProcessTrim(track, info, pkt_ref);
+    };
+
+  next_sched_time_ = LocalClock::now();
+  next_sched_time_known_ = true;
+}
+
+StandardOutputBase::~StandardOutputBase() {}
+
+void StandardOutputBase::Process() {
+  bool mixed = false;
+  LocalTime now = LocalClock::now();
+
+  // At this point, we should always know when our implementation would like to
+  // be called to do some mixing work next.  If we do not know, then we should
+  // have already shut down.
+  //
+  // If the next sched time has not arrived yet, don't attempt to mix anything.
+  // Just trim the queues and move on.
+  DCHECK(next_sched_time_known_);
+  if (now >= next_sched_time_) {
+    // Clear the flag, if the implementation does not set this flag by calling
+    // SetNextSchedTime during the cycle, we consider it to be an error and shut
+    // down.
+    next_sched_time_known_ = false;
+
+    // As long as our implementation wants to mix more and has not run into a
+    // problem trying to finish the mix job, mix some more.
+    do {
+      ::memset(&cur_mix_job_, 0, sizeof(cur_mix_job_));
+
+      if (!StartMixJob(&cur_mix_job_, now)) {
+        break;
+      }
+
+      ForeachTrack(setup_mix_, process_mix_);
+      mixed = true;
+    } while (FinishMixJob(cur_mix_job_));
+  }
+
+  if (!next_sched_time_known_) {
+    // TODO(johngro): log this as an error.
+    ShutdownSelf();
+    return;
+  }
+
+  // If we mixed nothing this time, make sure that we trim all of our track
+  // queues.  No matter what is going on with the output hardware, we are not
+  // allowed to hold onto the queued data past its presentation time.
+  if (!mixed) {
+    ForeachTrack(setup_trim_, process_trim_);
+  }
+
+  // Figure out when we should wake up to do more work again.  No matter how
+  // long our implementation wants to wait, we need to make sure to wake up and
+  // periodically trim our input queues.
+  LocalTime max_sched_time = now + MAX_TRIM_PERIOD;
+  ScheduleCallback((next_sched_time_ > max_sched_time)
+                   ? max_sched_time
+                   : next_sched_time_);
+}
+
+MediaResult StandardOutputBase::InitializeLink(
+    const AudioTrackToOutputLinkPtr& link) {
+  TrackBookkeeping* bk = AllocBookkeeping();
+  AudioTrackToOutputLink::BookkeepingPtr ref(bk);
+
+  // We should never fail to allocate our bookkeeping.  The only way this can
+  // happen is if we have a badly behaved implementation.
+  if (!bk) { return MediaResult::INTERNAL_ERROR; }
+
+  // We cannot proceed if our track has somehow managed to go away already.
+  AudioTrackImplPtr track = link->GetTrack();
+  if (!track) { return MediaResult::INVALID_ARGUMENT; }
+
+  // Pick a mixer based on the input and output formats.
+  bk->mixer = Mixer::Select(track->Format(), output_format_);
+  if (bk->mixer == nullptr) { return MediaResult::UNSUPPORTED_CONFIG; }
+
+  // Looks like things went well.  Stash a reference to our bookkeeping and get
+  // out.
+  link->output_bookkeeping() = std::move(ref);
+  return MediaResult::OK;
+}
+
+StandardOutputBase::TrackBookkeeping* StandardOutputBase::AllocBookkeeping() {
+  return new TrackBookkeeping();
+}
+
+void StandardOutputBase::ForeachTrack(const TrackSetupTask& setup,
+                                      const TrackProcessTask& process) {
+  for (auto iter = links_.begin(); iter != links_.end(); ) {

[jeffbrown, 2015/11/04 23:43:34]

I'm actually kind of surprised that you designed the mixer with an outer loop
over the tracks instead of over the outputs.

This means as we go through the tracks, we have to accumulate all of the samples
into the destination buffer which may cause a loss of precision and bad clipping
artifacts.

With that in mind, perhaps we should only support accumulation buffers that have
a normalized representation (say, int32_t), and convert them into the
destination representation at the very last moment.

That should make your Mix functions a little simpler since they won't have to
deal with converting the accumulated samples to a destination format
immediately.  They can do all of the math in the normalized domain.

Another alternative might be to try to exchange the order of the loops but I
suspect that the bookkeeping involved in doing that efficiently might be a
little much!

[johngro, 2015/11/06 02:20:27]

Keep in mind that the outputs are independent and mix in parallel.  They also
have completely independent timing/buffering requirements.  We never loop over
outputs.

That said, I suspect that picking a canonical output sample format, or small set
of them (int32_t/double) and separating the mix from the clip operation is a
likely direction that this will end up moving in.  Things get a bit more
complicated when 24 bit audio comes in (int32_t internal allows a max of 127
tracks before internal clipping becomes a possibility), and when
greater-than-unity gain is permitted on input tracks (something which Dale has
expressed interest in supporting).  At this point, we are looking at int64_t as
our intermediate format, and its starting to look seriously expensive.

TL;DR?  I expect that an intermediate mix buffer will be coming soon, its just
not here yet.

Finally; re: permuting the order of the loops..

If what you are trying to say is that you are puzzled why I would write
foreach(track : tracks_)
  foreach(packet : track.pending_packets_)

I'm not sure I see a way to permute the order of these loops.  The
pending_packet_ concept would need to be homogeneous across the loops and it is
not.  Even if it was, I would still go over tracks first in order to implement
priority properly (something which Carlos has requested).  His request (and I
think it is reasonable/good) is that if this starts to get so overloaded because
of the number of tracks in the system, that we cut-n-run during the mix
operation.  This means that low priority tracks go silent, but we do not
underflow our output DMA.  In the first model, we punish low priority tracks
when we are overloaded.  In the second, we punish all tracks.  Carlos asserts
that this is what Apple does and Microsoft does not, which is one of the reasons
he preferred the Apple audio system over the system offered by MSFT.

+    if (shutting_down()) { return; }
+
+    // Is the track still around?  If so, process it.  Otherwise, remove the
+    // track entry and move on.
+    const AudioTrackToOutputLinkPtr& link = *iter;
+    AudioTrackImplPtr track(link->GetTrack());
+
+    auto tmp_iter = iter++;
+    if (!track) {
+      links_.erase(tmp_iter);
+      continue;
+    }
+
+    // It would be nice to be able to use a dynamic cast for this, but currently
+    // we are building with no-rtti
+    TrackBookkeeping* info =
+      static_cast<TrackBookkeeping*>(link->output_bookkeeping().get());
+    DCHECK(info);
+
+    // Make sure that the mapping between the track's frame time domain and
+    // local time is up to date.
+    info->UpdateTrackTrans(track);
+
+    bool setup_done = false;
+    AudioPipe::AudioPacketRefPtr pkt_ref;
+    while (true) {
+      // Try to grab the front of the packet queue.  If it has been flushed
+      // since the last time we grabbed it, be sure to reset our mixer's
+      // internal filter state.
+      bool was_flushed;
+      pkt_ref = link->LockPendingQueueFront(&was_flushed);
+      if (was_flushed) {
+        info->mixer->Reset();
+      }
+
+      // If the queue is empty, then we are done.
+      if (!pkt_ref) { break; }
+
+      // If we have not set up for this track yet, do so.  If the setup fails
+      // for any reason, stop processing packets for this track.
+      if (!setup_done) {
+        setup_done = setup(track, info);
+        if (!setup_done) { break; }
+      }
+
+      // Now process the packet which is at the front of the track's queue.  If
+      // the packet has been entirely consumed, pop it off the front and proceed
+      // to the next one.  Otherwise, we are finished.
+      if (!process(track, info, pkt_ref)) { break; }
+      link->UnlockPendingQueueFront(&pkt_ref, true);
+    }
+
+    // Unlock the queue and proceed to the next track.
+    link->UnlockPendingQueueFront(&pkt_ref, false);
+
+    // Note: there is no point in doing this for the trim task, but it dosn't
+    // hurt anything, and its easier then introducing another function to the
+    // ForeachTrack arguments to run after each track is processed just for the
+    // purpose of setting this flag.
+    cur_mix_job_.accumulate = true;
+  }
+}
+
+bool StandardOutputBase::SetupMix(const AudioTrackImplPtr& track,
+                                  TrackBookkeeping* info) {
+  // If we need to recompose our transformation from output frame space to input
+  // fractional frames, do so now.
+  DCHECK(info);
+  info->UpdateOutputTrans(cur_mix_job_);
+  cur_mix_job_.frames_produced = 0;
+
+  return true;
+}
+
+bool StandardOutputBase::ProcessMix(
+    const AudioTrackImplPtr& track,
+    TrackBookkeeping* info,
+    const AudioPipe::AudioPacketRefPtr& pkt_ref) {
+  // Sanity check our parameters.
+  DCHECK(info);
+  DCHECK(pkt_ref);
+
+  // We had better have a valid job, or why are we here?
+  DCHECK(cur_mix_job_.buf);
+  DCHECK(cur_mix_job_.buf_frames);
+  DCHECK(cur_mix_job_.frames_produced <= cur_mix_job_.buf_frames);
+
+  // Have we produced all that we are supposed to?  If so, hold the current
+  // packet and move on to the next track.
+  if (cur_mix_job_.frames_produced >= cur_mix_job_.buf_frames) {
+    return false;
+  }
+
+  uint32_t frames_left = cur_mix_job_.buf_frames - cur_mix_job_.frames_produced;
+  void* buf = static_cast<uint8_t*>(cur_mix_job_.buf)
+            + (cur_mix_job_.frames_produced * output_bytes_per_frame_);
+
+  // Figure out where this job starts, expressed in fractional input frames.
+  int64_t start_pts_ftf;
+  bool good = info->out_frames_to_track_frames.DoForwardTransform(
+      cur_mix_job_.start_pts_of + cur_mix_job_.frames_produced,
+      &start_pts_ftf);
+  DCHECK(good);
+
+  // If the start of this mix job is past the end of this packet presentation,
+  // do no mixing.  Let the ForeachTrack loop know that we are done with the
+  // packet and it can be released.
+  if (start_pts_ftf >= pkt_ref->end_pts()) {
+    return true;
+  }
+
+  // If this track is currently paused (or being sampled extremely slowly), our
+  // step size will be zero.  We know that this packet will be relevant at some
+  // point in the future, but right now it contributes nothing.  Tell the
+  // ForeachTrack loop that we are done and to hold onto this packet for now.
+  if (!info->step_size) {
+    return false;
+  }
+
+  // Figure out how many output samples into the current job this packet starts.
+  int64_t delta;
+  int64_t output_offset_64;
+  if (pkt_ref->start_pts() > start_pts_ftf) {
+    delta             = pkt_ref->start_pts() - start_pts_ftf;
+    output_offset_64  = delta + info->step_size - 1;
+    output_offset_64 /= info->step_size;
+  } else {
+    output_offset_64 = 0;
+  }
+  DCHECK_GE(output_offset_64, 0);
+
+  // If this packet starts after the end of this job (entirely in the future),
+  // then we are done for now.
+  if (output_offset_64 >= frames_left) {
+    return false;
+  }
+
+  // Figure out the offset (in fractional frames) into this packet where we want
+  // to start sampling.
+  int64_t  input_offset_64;
+  if (output_offset_64) {
+    input_offset_64  = output_offset_64 * info->step_size;
+    input_offset_64 -= delta;
+    DCHECK_LT(input_offset_64, info->step_size);
+  } else {
+    input_offset_64 = start_pts_ftf - pkt_ref->start_pts();
+  }
+  DCHECK_GE(input_offset_64, 0);
+  DCHECK_LE(input_offset_64, std::numeric_limits<int32_t>::max());
+  DCHECK_LT(input_offset_64, pkt_ref->end_pts() - pkt_ref->start_pts());
+
+  uint32_t input_offset  = static_cast<uint32_t>(input_offset_64);
+  uint32_t output_offset = static_cast<uint32_t>(output_offset_64);
+  const auto& regions = pkt_ref->regions();
+  DCHECK(info->mixer != nullptr);
+
+  for (size_t i = 0;
+      (i < regions.size()) && (output_offset < frames_left);
+      ++i) {
+    const auto& region = regions[i];
+
+    if (input_offset >= region.frac_frame_len) {
+      input_offset -= region.frac_frame_len;
+      continue;
+    }
+
+    bool consumed_source = info->mixer->Mix(buf,

[jeffbrown, 2015/11/04 23:43:34]

As designed, we're going to have problems replacing the point sampler with one
that does interpolation.  An interpolating sampler may need access to data that
comes from multiple regions within the packet (or from prior packets depending
on how the audio got sliced up).  In order words, the sampler will need access
to a window of samples within the track, not just one.

To work around this problem, consider passing the mix function a random access
iterator for accessing the Nth sample of the input rather than a pointer to it. 
You'll have to define what the window size is upfront though.  The window might
need to depend on properties of the timeline transformation.

Alternately, it's possible that copying a moving window's worth of the input
into a linear buffer will actually end up being faster than trying to process it
in small chunks when crossing region boundaries.  (Best of both worlds, assuming
the windows are small compared to typical regions so we don't have to deal with
overlap very often.)

[johngro, 2015/11/06 02:20:27]

Acknowledged.

This is a complicated optimization subject.  I don't know where we will
eventually end up, but right now I am trying to pursue a model where we minimize
copies (one of Dale's primary concerns).  I have a WiP of a linear interpolation
resampler which holds just the last sample from the last region/packet, and
resamples in place for everything else.  As filters grow wider, and buffers grow
smaller (to hit extreme low latency requirements) the overhead of handling the
edges of the system is going to start to grow, however.  Pursuing a strategy of
copying into a temporally contiguous input buffer may become the most efficient
thing to do.  This could also serve to allow simpler implementations of graceful
ramp down/up after/before discontinuities and still preserve the ability to
allow sparse audio, something that can really benefit an application like the
announcer.

We'll see what the future holds.  Generally, optimizing the core mix loops is
going to be a full-time job for someone someday soon.

+                                            frames_left,
+                                            &output_offset,
+                                            region.base,
+                                            region.frac_frame_len,
+                                            &input_offset,
+                                            info->step_size,
+                                            cur_mix_job_.accumulate);
+    DCHECK_LE(output_offset, frames_left);
+
+    if (!consumed_source) {
+      // Looks like we didn't consume all of this region.  Assert that we have
+      // produced all of our frames and we are done.
+      DCHECK(output_offset == frames_left);
+      return false;
+    }
+
+    input_offset -= region.frac_frame_len;
+  }
+
+  cur_mix_job_.frames_produced += output_offset;
+  DCHECK(cur_mix_job_.frames_produced <= cur_mix_job_.buf_frames);
+  return true;
+}
+
+bool StandardOutputBase::SetupTrim(const AudioTrackImplPtr& track,
+                                   TrackBookkeeping* info) {
+  // Compute the cutoff time we will use to decide wether or not to trim
+  // packets.  ForeachTracks has already updated our transformation, no need
+  // for us to do so here.
+  DCHECK(info);
+
+  int64_t local_now_ticks = LocalClock::now().time_since_epoch().count();
+
+  // The behavior of the RateControlBase implementation guarantees that the
+  // transformation into the media timeline is never singular.  If the
+  // forward transformation fails it can only be because of an overflow,
+  // which should be impossible unless the user has defined a playback rate
+  // where the ratio between media time ticks and local time ticks is
+  // greater than one.
+  //
+  // IOW - this should never happen.  If it does, we just stop processing
+  // payloads.
+  //
+  // TODO(johngro): Log an error?  Communicate this to the user somehow?
+  if (!info->lt_to_track_frames.DoForwardTransform(local_now_ticks,
+                                                  &trim_threshold_)) {
+    return false;
+  }
+
+  return true;
+}
+
+bool StandardOutputBase::ProcessTrim(
+    const AudioTrackImplPtr& track,
+    TrackBookkeeping* info,
+    const AudioPipe::AudioPacketRefPtr& pkt_ref) {
+  DCHECK(pkt_ref);
+
+  // If the presentation end of this packet is in the future, stop trimming.
+  if (pkt_ref->end_pts() > trim_threshold_) {
+    return false;
+  }
+
+  return true;
+}
+
+void StandardOutputBase::TrackBookkeeping::UpdateTrackTrans(
+    const AudioTrackImplPtr& track) {
+  LinearTransform tmp;
+  uint32_t gen;
+
+  DCHECK(track);
+  track->SnapshotRateTrans(&tmp, &gen);
+
+  // If the local time -> media time transformation has not changed since the
+  // last time we examines it, just get out now.
+  if (lt_to_track_frames_gen == gen) { return; }
+
+  // The transformation has changed, re-compute the local time -> track frame
+  // transformation.
+  LinearTransform scale(0, track->FractionalFrameToMediaTimeRatio(), 0);
+  bool good;
+
+  lt_to_track_frames.a_zero = tmp.a_zero;
+  good = scale.DoReverseTransform(tmp.b_zero, &lt_to_track_frames.b_zero);
+  DCHECK(good);
+  good = LinearTransform::Ratio::Compose(scale.scale,
+                                         tmp.scale,
+                                         &lt_to_track_frames.scale);
+  DCHECK(good);
+
+  // Update the generation, and invalidate the output to track generation.
+  lt_to_track_frames_gen = gen;
+  out_frames_to_track_frames_gen = MixJob::INVALID_GENERATION;
+}
+
+void StandardOutputBase::TrackBookkeeping::UpdateOutputTrans(
+    const MixJob& job) {
+  // We should not be here unless we have a valid mix job.  From our point of
+  // view, this means that we have a job which supplies a valid transformation
+  // from local time to output frames.
+  DCHECK(job.local_to_output);
+  DCHECK(job.local_to_output_gen != MixJob::INVALID_GENERATION);
+
+  // If our generations match, we don't need to re-compute anything.  Just use
+  // what we have already.
+  if (out_frames_to_track_frames_gen == job.local_to_output_gen) { return; }
+
+  // Assert that we have a good mapping from local time to fractional track
+  // frames.
+  //
+  // TODO(johngro): Don't assume that 0 means invalid.  Make it a proper
+  // constant defined somewhere.
+  DCHECK(lt_to_track_frames_gen);
+
+  // Compose the job supplied transformation from local to output with the
+  // track supplied mapping from local to fraction input frames to produce a
+  // transformation which maps from output frames to fractional input frames.
+  //
+  // TODO(johngro): Make this composition operation part of the LinearTransform
+  // class instead of doing it by hand here.  Its a more complicated task that
+  // one might initially think, because of the need to deal with the
+  // intermediate offset term, and distributing it to either side of the end of
+  // the transformation with a minimum amt of loss, while avoiding overflow.
+  //
+  // For now, we punt, do it by hand and just assume that everything went well.
+  LinearTransform& dst = out_frames_to_track_frames;
+
+  // Distribute the intermediate offset entirely to the fractional frame domain
+  // for now.  We can do better by extracting portions of the intermedate
+  // offset that can be scaled by the ratios on either side of with without
+  // loss, but for now this should be close enough.
+  int64_t intermediate = job.local_to_output->a_zero
+                       - lt_to_track_frames.a_zero;
+  int64_t track_frame_offset;
+
+  // TODO(johngro): add routines to LinearTransform::Ratio which allow us to
+  // scale using just a ratio without needing to create a linear transform with
+  // empty offsets.
+  LinearTransform tmp(0, lt_to_track_frames.scale, 0);
+  bool good = tmp.DoForwardTransform(intermediate, &track_frame_offset);
+  DCHECK(good);
+
+  dst.a_zero = job.local_to_output->b_zero;
+  dst.b_zero = lt_to_track_frames.b_zero + track_frame_offset;
+
+  // TODO(johngro): Add options to allow us to invert one or both of the ratios
+  // during composition instead of needing to make a temporary ratio to
+  // acomplish the task.
+  LinearTransform::Ratio tmp_ratio(job.local_to_output->scale.denominator,
+                                   job.local_to_output->scale.numerator);
+  good = LinearTransform::Ratio::Compose(tmp_ratio,
+                                         lt_to_track_frames.scale,
+                                         &dst.scale);;
+  DCHECK(good);
+
+  // Finally, compute the step size in fractional frames.  IOW, every time we
+  // move forward one output frame, how many fractional frames of input do we
+  // consume.  Don't bother doing the multiplication if we already know that the
+  // numerator is zero.
+  //
+  // TODO(johngro): same complaint as before... Do this without a temp.  The
+  // special casing should be handled in the routine added to
+  // LinearTransform::Ratio.
+  DCHECK(dst.scale.denominator);
+  if (!dst.scale.numerator) {
+    step_size = 0;
+  } else {
+    LinearTransform tmp(0, dst.scale, 0);
+    int64_t tmp_step_size;
+
+    good = tmp.DoForwardTransform(1, &tmp_step_size);
+
+    DCHECK(good);
+    DCHECK_GE(tmp_step_size, 0);
+    DCHECK_LE(tmp_step_size, std::numeric_limits<uint32_t>::max());
+
+    step_size = static_cast<uint32_t>(tmp_step_size);
+  }
+
+  // Done, update our generation.
+  out_frames_to_track_frames_gen = job.local_to_output_gen;
+}
+
+}  // namespace audio
+}  // namespace media
+}  // namespace mojo