Automatic resolution throttling for screen capture pipeline.

media/capture/video_capture_oracle.h

Index: media/capture/video_capture_oracle.h
diff --git a/media/capture/video_capture_oracle.h b/media/capture/video_capture_oracle.h
index 8e2980dab0a0aba8e9402f113d5375c6f7680fa8..7d0087a86add86b4d477f899ea844bd28af46f7d 100644
--- a/media/capture/video_capture_oracle.h
+++ b/media/capture/video_capture_oracle.h
@@ -10,6 +10,8 @@
 #include "base/time/time.h"
 #include "media/base/media_export.h"
 #include "media/capture/animated_content_sampler.h"
+#include "media/capture/capture_resolution_chooser.h"
+#include "media/capture/feedback_signal_accumulator.h"
 #include "media/capture/smooth_event_sampler.h"
 #include "ui/gfx/geometry/rect.h"
 
@@ -34,9 +36,17 @@ class MEDIA_EXPORT VideoCaptureOracle {
     kMinTimerPollPeriodMillis = 125,  // 8 FPS
   };
 
-  explicit VideoCaptureOracle(base::TimeDelta min_capture_period);
+  VideoCaptureOracle(base::TimeDelta min_capture_period,
+                     const gfx::Size& max_frame_size,
+                     media::ResolutionChangePolicy resolution_change_policy);
+
   virtual ~VideoCaptureOracle();
 
+  // Sets the source content size.  This may not have an immediate affect on the

[hubbe, 2015/06/23 20:04:31]

affect -> effect

[miu, 2015/06/25 20:48:59]

Done.  On a Dvorak keyboard, that's a typo.  ;-)

+  // proposed capture size, as the oracle will prevent too-frequent changes from
+  // occurring.
+  void SetSourceSize(const gfx::Size& source_size);
+
   // Record a event of type |event|, and decide whether the caller should do a
   // frame capture.  |damage_rect| is the region of a frame about to be drawn,
   // and may be an empty Rect, if this is not known.  If the caller accepts the
@@ -45,9 +55,14 @@ class MEDIA_EXPORT VideoCaptureOracle {
                                     const gfx::Rect& damage_rect,
                                     base::TimeTicks event_time);
 
-  // Record the start of a capture.  Returns a frame_number to be used with
-  // CompleteCapture().
-  int RecordCapture();
+  // Record and update internal state based on whether the frame capture will be
+  // started.  |pool_utilization| is a value in the range 0.0 to 1.0 to indicate
+  // the current buffer pool utilization relative to a sustainable maximum.
+  // This method should only be called if the last call to
+  // ObserveEventAndDecideCapture() returned true.  The first method returns the
+  // |frame_number| to be used with CompleteCapture().
+  int RecordCapture(double pool_utilization);
+  void RecordWillNotCapture(double pool_utilization);
 
   // Notify of the completion of a capture, and whether it was successful.
   // Returns true iff the captured frame should be delivered.  |frame_timestamp|
@@ -57,6 +72,14 @@ class MEDIA_EXPORT VideoCaptureOracle {
                        bool capture_was_successful,
                        base::TimeTicks* frame_timestamp);
 
+  // Record the resource utilization feedback for a frame that was processed by
+  // the consumer.  This allows the oracle to reduce/increase future data volume
+  // if the consumer is over/underloaded.  |resource_utilization| is a value in

[hubbe, 2015/06/23 20:04:31]

over/underloaded -> overloaded/underutilized ?

[miu, 2015/06/25 20:49:00]

Done.

+  // the range 0.0 to 1.0 to indicate the current consumer resource utilization
+  // relative to a sustainable maximum.  This method should only be called for
+  // frames where CompleteCapture() returned true.
+  void RecordConsumerFeedback(int frame_number, double resource_utilization);
+
   base::TimeDelta min_capture_period() const {
     return smoothing_sampler_.min_capture_period();
   }
@@ -68,11 +91,48 @@ class MEDIA_EXPORT VideoCaptureOracle {
     return duration_of_next_frame_;
   }
 
+  // Returns the capture frame size the client should use.  This is updated by
+  // calls to ObserveEventAndDecideCapture().  The oracle prevents too-frequent
+  // changes to the capture size, to avoid stressing the end-to-end pipeline.
+  gfx::Size capture_size() const { return capture_size_; }
+
  private:
-  // Retrieve/Assign a frame timestamp by capture |frame_number|.
+  // Retrieve/Assign a frame timestamp by capture |frame_number|.  Only valid
+  // when IsFrameInRecentHistory(frame_number) returns true.
   base::TimeTicks GetFrameTimestamp(int frame_number) const;
   void SetFrameTimestamp(int frame_number, base::TimeTicks timestamp);
 
+  // Returns true if the frame timestamp ring-buffer currently includes a
+  // slot for the given |frame_number|.
+  bool IsFrameInRecentHistory(int frame_number) const;
+
+  // Queries the ResolutionChooser to update |capture_size_|, and resets all the
+  // FeedbackSignalAccumulator members to stable-state starting values.  The
+  // accumulators are reset such that they can only apply feedback updates for
+  // future frames (those with a timestamp after |last_frame_time|).
+  void CommitCaptureSizeAndReset(base::TimeTicks last_frame_time);
+
+  // Resets the accumulators that are used to determine when to increase the
+  // capture size.  This method is called whenever something happens that should
+  // re-start the "proving period," to prove that the capture size can be safely
+  // increased.
+  void ResetPessimisticAccumulators(base::TimeTicks event_time);
+
+  // Called after a capture or no-capture decision was recorded.  This analyzes
+  // current state and may result in a future change to the capture frame size.
+  void AnalyzeAndAdjust(base::TimeTicks analyze_time);
+
+  // Analyzes current feedback signal accumulators for an indication that the
+  // capture size should be decreased.  Returns either a decreased area, or -1
+  // if no decrease should be made.
+  int AnalyzeForDecreasedArea(base::TimeTicks analyze_time);
+
+  // Analyzes current feedback signal accumulators for an indication that the
+  // the system has had excellent long-term performance and the capture size
+  // should be increased to improve quality.  Returns either an increased area,
+  // or -1 if no increase should be made.
+  int AnalyzeForIncreasedArea(base::TimeTicks analyze_time);
+
   // Incremented every time a paint or update event occurs.
   int next_frame_number_;
 
@@ -98,11 +158,41 @@ class MEDIA_EXPORT VideoCaptureOracle {
   SmoothEventSampler smoothing_sampler_;
   AnimatedContentSampler content_sampler_;
 
+  // Determines video capture frame sizes.
+  CaptureResolutionChooser resolution_chooser_;
+
+  // The current capture size.  |resolution_chooser_| may hold an updated value
+  // because the oracle prevents this size from changing too frequently.  This
+  // avoids over-stressing consumers (e.g., when a window is being activly
+  // drag-resized) and allowing the end-to-end system time to stabilize.
+  gfx::Size capture_size_;
+
   // Recent history of frame timestamps proposed by VideoCaptureOracle.  This is
   // a ring-buffer, and should only be accessed by the Get/SetFrameTimestamp()
   // methods.
   enum { kMaxFrameTimestamps = 16 };
   base::TimeTicks frame_timestamps_[kMaxFrameTimestamps];
+
+  // Recent average buffer pool utilization for capture.
+  FeedbackSignalAccumulator buffer_pool_utilization_;
+
+  // Estimated maximum frame area that currently can be handled by the consumer,
+  // in number of pixels per frame.  This is used to adjust the capture size up
+  // or down to a data volume the consumer can handle.  Note that some consumers
+  // do not provide feedback, and the analysis logic should account for that.
+  FeedbackSignalAccumulator estimated_capable_area_;
+
+  // The timestamp of the frame where |content_sampler_| last detected
+  // animation.  This determines whether capture size increases will be
+  // aggressive (because content is not animating).
+  base::TimeTicks last_time_animation_was_detected_;
+
+  // These accumulate the same data points as |buffer_pool_utilization_| and
+  // |estimated_capable_area_|, but accumulate a much longer-term time-weighted
+  // average.  These are used to judge whether capture size increases are
+  // warranted.
+  FeedbackSignalAccumulator pessimistic_pool_utilization_;
+  FeedbackSignalAccumulator pessimistic_capable_area_;
 };
 
 }  // namespace media