Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(261)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: tracing/tracing/metrics/webrtc/webrtc_rendering_metric_test.html

Issue 2711623002: Add a TBMv2 webrtc_rendering_metric. (Closed)
Patch Set: Customize summary options. Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« tracing/tracing/metrics/webrtc/webrtc_rendering_metric.html ('K') | « tracing/tracing/metrics/webrtc/webrtc_rendering_metric.html ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tracing/tracing/metrics/webrtc/webrtc_rendering_metric_test.html
diff --git a/tracing/tracing/metrics/webrtc/webrtc_rendering_metric_test.html b/tracing/tracing/metrics/webrtc/webrtc_rendering_metric_test.html
new file mode 100644
index 0000000000000000000000000000000000000000..9d1b0c413151be66c53108882dc5b2d42b921ebc
--- /dev/null
+++ b/tracing/tracing/metrics/webrtc/webrtc_rendering_metric_test.html
@@ -0,0 +1,450 @@
+<!DOCTYPE html>
+<!--
+Copyright 2017 The Chromium Authors. All rights reserved.
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+-->
+
+<link rel="import" href="/tracing/core/test_utils.html">
+<link rel="import" href="/tracing/extras/importer/trace_event_importer.html">
+<link rel="import" href="/tracing/metrics/webrtc/webrtc_rendering_metric.html">
+<link rel="import" href="/tracing/model/slice_group.html">
+<link rel="import" href="/tracing/value/histogram.html">
+<link rel="import" href="/tracing/value/histogram_set.html">
+
+<script>
+'use strict';
+
+tr.b.unittest.testSuite(function() {
+ const DISPLAY_HERTZ = 60.0;
+ const VSYNC_DURATION_US = 1e6 / DISPLAY_HERTZ;
+
+ /**
+ * @param {Array.<Number>} pair - An array of length 2, from which a valid
+ * WebMediaPlayerMS event will be generated.
+ * @returns {event} A valid WebMediaPlayerMS event where the Ideal Render
+ * Instant is the first element and the Actual Render Begin is the second
+ * element.
+ */
+ function eventFromPair(pair) {
+ return {
+ title: 'WebMediaPlayerMS::UpdateCurrentFrame',
+ start: pair[1],
+ duration: 1,
+ args: {
+ 'Ideal Render Instant': pair[0],
+ 'Actual Render Begin': pair[1],
+ 'Actual Render End': 0,
+ 'Serial': 0,
+ }
+ };
+ }
+
+ /**
+ * @param {Array.<Number>} driftTimes - An array with the desired driftTimes.
+ * @return {Array.<event>} An array of events such that the drift times
+ * computed by webrtcRenderingMetric is the same as driftTimes.
+ */
+ function eventsFromDriftTimes(driftTimes) {
+ let pairs = [];
+ for (let i = 1; i <= driftTimes.length; ++i) {
+ pairs.push([i, i + driftTimes[i - 1]]);
+ }
+ return pairs.map(eventFromPair);
+ }
+
+ /**
+ * @param {Array.<Number>} normDriftTimes - To decide if a frame is out of
+ * sync or badly out of sync, we use the normalized drift times, that we get
+ * by subtracting the mean from each entry of the drift times array. The sum
+ * of the normDriftTimes must equal 0.
+ * @return {Array.<event>) An array of events such that when we normalize the
+ * drift times computed by webrtcRenderingMetric, we get the normDriftTimes
+ * array.
+ */
+ function eventsFromNormDriftTimes(normDriftTimes) {
+ /* Let
+ * B[i] = normDriftTimes[i]
+ * A[i] = driftTimes[i] be the array we want to find.
+ *
+ * We require that:
+ * sum(B[i]) = 0
+ *
+ * Then
+ * B[i] = A[i] - mean(A)
+ * => B[i] - B[0] = A[i] - mean(A) - A[0] + mean(A)
+ * => B[i] - B[0] = A[i] - A[0]
+ * => A[i] = B[i] - B[0] + A[0]
+ *
+ * We can fix A[0] to any number we want.
+ *
+ * Let's make sure that the array A we found generates the array B when
+ * normalized:
+ * A[i] - mean(A)
+ * = A[i] - sum(A[j]) / n
+ * = B[i] - B[0] + A[0] - sum(B[j] - B[0] + A[0]) / n
+ * = B[i] - B[0] + A[0] - (sum(B[j]) - n B[0] / n + n A[0] / n)
+ * = B[i] - B[0] + A[0] - sum(B[j]) + B[0] - A[0]
+ * = B[i] - sum(B[j])
+ * = B[i] since we require sum(B[j]) = 0
+ */
+ let driftTimes = [10000];
+ for (let i = 1; i < normDriftTimes.length; ++i) {
+ driftTimes.push(normDriftTimes[i] - normDriftTimes[0] + driftTimes[0]);
+ }
+ return eventsFromDriftTimes(driftTimes);
+ }
+
+ /**
+ * @param {Array.<Array.<Number>>} frameDistribution - An array of pairs
+ * encoding the source to output distribution. That is an array where each
+ * [ticks, count] entry says that there are 'count' frames that are displayed
+ * 'ticks' times.
+ * @returns {Array.<events>} The events that give rise to the given
+ * frameDistribution.
+ */
+ function eventsFromFrameDistribution(frameDistribution) {
+ let frameId = 0;
+ let pairs = [];
+ for (let [ticks, count] of frameDistribution) {
+ // We need 'count' runs, each run consisting of 'ticks' repeated elements.
+ for (let i = 0; i < count; ++i) {
+ frameId += 1;
+ for (let j = 0; j < ticks; ++j) {
+ // Frames are decided by the Ideal Render Instant.
+ pairs.push([frameId, 0]);
+ }
+ }
+ }
+ return pairs.map(eventFromPair);
+ }
+
+ function newModel(fakeEvents) {
+ function customizeModelCallback(model) {
+ const rendererProcess = model.getOrCreateProcess(1);
+ const mainThread = rendererProcess.getOrCreateThread(2);
+ for (const event of fakeEvents) {
+ mainThread.sliceGroup.pushSlice(tr.c.TestUtils.newSliceEx(event));
+ }
+ }
+ return tr.c.TestUtils.newModelWithEvents([], {customizeModelCallback});
+ }
+
+ function runWebrtcRenderingMetric(fakeEvents) {
+ let histograms = new tr.v.HistogramSet();
+ let model = newModel(fakeEvents);
+ tr.metrics.webrtc.webrtcRenderingMetric(histograms, model);
+ return histograms;
+ }
+
+ test('frameDistribution', function() {
+ // These numbers don't mean anything, we just want to make sure we can
+ // recover them after running webrtcRenderingMetric.
+ let frameDistribution = [[10, 3], [5, 15], [3, 146], [1, 546], [2, 10]];
+ let frameHist = new tr.v.Histogram('', tr.b.Unit.byName.unitlessNumber);
+ for (const [ticks, count] of frameDistribution) {
+ for (let i = 0; i < count; ++i) {
+ frameHist.addSample(ticks);
+ }
+ }
+ let fakeEvents = eventsFromFrameDistribution(frameDistribution);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ // We don't have access to the values stored in the histogram, so we check
+ // for equality in the summary statistics.
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_frame_distribution');
+ assert.strictEqual(hist.sum, frameHist.sum);
+ assert.strictEqual(hist.numValues, frameHist.numValues);
+ assert.strictEqual(hist.running.min, frameHist.running.min);
+ assert.strictEqual(hist.running.max, frameHist.running.max);
+ assert.closeTo(hist.standardDeviation, frameHist.standardDeviation, 1e-2);
+ });
+
+ test('driftTime', function() {
+ // These numbers don't mean anything. We just want to make sure we can
+ // recover them after running the metric.
+ let fakeDriftTimes = [16700, 17640, 15000, 24470, 16700, 14399, 17675];
+ let fakeEvents = eventsFromDriftTimes(fakeDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ // We don't have access to the values stored in the histogram, so we check
+ // for equality in the summary statistics.
+ let hist = histograms.getHistogramNamed('WebRTCRendering_drift_time');
+ assert.strictEqual(hist.sum, tr.b.Statistics.sum(fakeDriftTimes));
+ assert.strictEqual(hist.numValues, fakeDriftTimes.length);
+ assert.strictEqual(hist.running.min, tr.b.Statistics.min(fakeDriftTimes));
+ assert.strictEqual(hist.running.max, tr.b.Statistics.max(fakeDriftTimes));
+ assert.closeTo(hist.standardDeviation,
+ tr.b.Statistics.stddev(fakeDriftTimes), 1e-2);
+ });
+
+ test('framesBadlyOutOfSyncPerfect', function() {
+ // None of these will exceed the threshold for badly out of sync events,
+ // which is about 33 333.
+ let normDriftTimes = [-16700, 17640, 15000, -17640, -15000, 16700];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_frames_badly_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 0);
+ });
+
+ test('framesBadylOutOfSync', function() {
+ // Only 34 000 will exceed the threshold for badly out of sync events,
+ // which is about 33 333.
+ let normDriftTimes = [-34000, 10000, 10000, 10000, 4000];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_frames_badly_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 1);
+ });
+
+ test('framesOutOfSyncPerfect', function() {
+ // None of these will exceed the threshold for badly out of sync, which is
+ // about 16 667.
+ let normDriftTimes = [-16600, 15640, 15000, -15640, -15000, 16600];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_frames_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 0);
+ });
+
+ test('framesOutOfSync', function() {
+ // Only 17000 will exceed the threshold for badly out of sync, which is
+ // about 16 667.
+ let normDriftTimes = [-17000, 5000, 5000, 5000, 2000];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_frames_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 1);
+ });
+
+ test('percentBadlyOutOfSyncPerfect', function() {
+ // None of these will exceed the threshold for badly out of sync events,
+ // which is about 33 333.
+ let normDriftTimes = [-16700, 17640, 15000, -17640, -15000, 16700];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_percent_badly_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 0);
+ });
+
+ test('percentBadylOutOfSync', function() {
+ // Only 34 000 will exceed the threshold for badly out of sync events,
+ // which is about 33 333.
+ let normDriftTimes = [-34000, 10000, 10000, 10000, 4000];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_percent_badly_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, .2);
+ });
+
+ test('percentOutOfSyncPerfect', function() {
+ // None of these will exceed the threshold for badly out of sync, which is
+ // about 16 667.
+ let normDriftTimes = [-16600, 15640, 15000, -15640, -15000, 16600];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_percent_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 0);
+ });
+
+ test('percentOutOfSync', function() {
+ // Only 17000 will exceed the threshold for badly out of sync, which is
+ // about 16 667.
+ let normDriftTimes = [-17000, 5000, 5000, 5000, 2000];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_percent_out_of_sync');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, .2);
+ });
+
+ test('smoothnessScorePerfect', function() {
+ // None of these will exceed the threshold for badly out of sync, which is
+ // about 16 667, so the smoothnessScore wil be perfect.
+ let normDriftTimes = [-16600, 15640, 15000, -15640, -15000, 16600];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist = histograms.getHistogramNamed('WebRTCRendering_smoothness_score');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 1);
+ });
+
+ test('smoothnessScore', function() {
+ // One will exceed the threshold for frames badly out of sync (33 333) and
+ // another two the threshold for frames out of sync (16 667). So the
+ // smoothness score is
+ // 1 - (frames out of sync + 3 * frames badly out of sync) / n
+ // = 1 - (2 + 3) / 5 = 0
+ let normDriftTimes = [-17000, 34000, -17000, -10000, 10000];
+ assert.strictEqual(tr.b.Statistics.sum(normDriftTimes), 0);
+ let fakeEvents = eventsFromNormDriftTimes(normDriftTimes);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist = histograms.getHistogramNamed('WebRTCRendering_smoothness_score');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 0);
+ });
+
+ test('fpsPerfect', function() {
+ // Every frame is displayed once. This is a perfect FPS of 60.
+ let frameDistribution = [[1, 10]];
+ let fakeEvents = eventsFromFrameDistribution(frameDistribution);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist = histograms.getHistogramNamed('WebRTCRendering_fps');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 60);
+ });
+
+ test('fps', function() {
+ // Every frame is displayed 15 times. This means an FPS of 4.
+ let frameDistribution = [[15, 10]];
+ let fakeEvents = eventsFromFrameDistribution(frameDistribution);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist = histograms.getHistogramNamed('WebRTCRendering_fps');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 4);
+ });
+
+ test('frozenFramesCountPerfect', function() {
+ // 10 frames are displayed one time, and 10 frames are displayed twice.
+ // This means no frames exceed the threshold of 6, and so no frames are
+ // considered frozen.
+ let frameDistribution = [[1, 10], [2, 10]];
+ let fakeEvents = eventsFromFrameDistribution(frameDistribution);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_frozen_frames_count');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 0);
+ });
+
+ test('frozenFramesCount', function() {
+ // 82 frames are displayed 1 time, 5 frames are displayed 2 times,
+ // and 1 frame is displayed 6 times.
+ // Only the drame displayed 6 times satisfies the threshold of 6. Since the
+ // first appearance is not considered frozen, there are 5 frozen frames.
+ let frameDistribution = [[1, 82], [2, 5], [6, 1]];
+ let fakeEvents = eventsFromFrameDistribution(frameDistribution);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_frozen_frames_count');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 5);
+ });
+
+ test('freezingScorePerfect', function() {
+ // Every frame is displayed 1 times. This means a perfect freezing score of
+ // 100.
+ let frameDistribution = [[1, 10]];
+ let fakeEvents = eventsFromFrameDistribution(frameDistribution);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist = histograms.getHistogramNamed('WebRTCRendering_freezing_score');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, 1);
+ });
+
+ test('freezingScore', function() {
+ // 82 frames are displayed 1 time, 5 frames are displayed 2 times,
+ // and 1 frame is displayed 8 times.
+ // This means that the total number of frames displayed is
+ // 82 * 1 + 5 * 2 + 1 * 8 = 100
+ // And the freezing score is
+ // 1 - 82 / 100 * weight[0]
+ // - 5 / 100 * weight[1]
+ // - 1 / 100 * weight[7]
+ // = 1 - .82 * 0 since weight[0] = 0
+ // - .05 * 0 since weight[1] = 0 too
+ // - .01 * 15 since weight[7] = 15
+ // = 1 - .15 = .85
+ // See frozenPenaltyWeight for information on the weights and
+ // addFreezingScore for the definition of the freezingScore.
+ let frameDistribution = [[1, 82], [2, 5], [8, 1]];
+ let fakeEvents = eventsFromFrameDistribution(frameDistribution);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist = histograms.getHistogramNamed('WebRTCRendering_freezing_score');
+ assert.strictEqual(hist.numValues, 1);
+ assert.strictEqual(hist.running.mean, .85);
+ });
+
+ test('renderingLengthErrorPerfect', function() {
+ let fakePairs = [];
+ for (let i = 1; i < 10; ++i) {
+ // Each frame's Ideal Render Instant is exactly VSYNC_DURATION_US after
+ // the previous one, so that the rendering length error is 0.
+ fakePairs.push([VSYNC_DURATION_US * i, 0]);
+ }
+ let fakeEvents = fakePairs.map(eventFromPair);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_rendering_length_error');
+ assert.strictEqual(hist.numValues, 1);
+ assert.closeTo(hist.running.mean, 0, 1e-3);
+ });
+
+ test('renderingLengthError', function() {
+ let errors = [1000, 3000, 0, 5000, 0, 2000];
+ let fakePairs = [[1, 0]];
+ for (let i = 0; i < errors.length; ++i) {
+ // Each frame's Ideal Render Instant is close to VSYNC_DURATION_US after
+ // the previous one, but with a known delay.
+ fakePairs.push([fakePairs[i][0] + VSYNC_DURATION_US + errors[i], 0]);
+ }
+
+ // The rendering length error is then the sum of the errors, normalized by
+ // the span between the first and the last Ideal Render Instants.
+ let idealRenderSpan = fakePairs[fakePairs.length - 1][0] - fakePairs[0][0];
+ let expectedRenderingLengthError = tr.b.Statistics.sum(errors) /
+ idealRenderSpan;
+
+ let fakeEvents = fakePairs.map(eventFromPair);
+ let histograms = runWebrtcRenderingMetric(fakeEvents);
+
+ let hist =
+ histograms.getHistogramNamed('WebRTCRendering_rendering_length_error');
+ assert.strictEqual(hist.numValues, 1);
+ assert.closeTo(hist.running.mean, expectedRenderingLengthError, 1e-3);
+ });
+});
+</script>
« tracing/tracing/metrics/webrtc/webrtc_rendering_metric.html ('K') | « tracing/tracing/metrics/webrtc/webrtc_rendering_metric.html ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698