WebRtcStatsPerfBrowserTest added, a perf test using the new getStats

chrome/browser/media/webrtc/webrtc_performance_browsertest.cc

Index: chrome/browser/media/webrtc/webrtc_performance_browsertest.cc
diff --git a/chrome/browser/media/webrtc/webrtc_performance_browsertest.cc b/chrome/browser/media/webrtc/webrtc_performance_browsertest.cc
new file mode 100644
index 0000000000000000000000000000000000000000..e26656af13362dd6cbb90d504d8d2187e50a6cf5
--- /dev/null
+++ b/chrome/browser/media/webrtc/webrtc_performance_browsertest.cc
@@ -0,0 +1,197 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+
+#include "base/command_line.h"
+#include "base/test/test_timeouts.h"
+#include "chrome/browser/media/webrtc/rtc_stats_dictionary.h"
+#include "chrome/browser/media/webrtc/webrtc_browsertest_base.h"
+#include "chrome/browser/media/webrtc/webrtc_browsertest_common.h"
+#include "content/public/common/content_switches.h"
+#include "content/public/common/feature_h264_with_openh264_ffmpeg.h"
+#include "media/base/media_switches.h"
+#include "testing/perf/perf_test.h"
+
+namespace content {
+
+namespace {
+
+const char kMainWebrtcTestHtmlPage[] = "/webrtc/webrtc_jsep01_test.html";
+

[phoglund_chromium, 2016/12/01 15:42:30]

Bytes received/sent is probably one of the more volatile stats you can get out.
You run for 60 seconds and then start looking for stats. What you'll be adding
at that point is

[10000000003, 10000000013, 10000000050, ...]

since many bytes have been already been sent. If the call gets established just
a little bit slower or faster it will have a big effect on the average, right?
As you can see on the main tests, they're really noisy (and I think they're
taken off monitoring for that reason:
https://chromeperf.appspot.com/report?sid=edbc402a707f488c03a4e799c1e84662d21...)

Are there any other stats could look at here? Otherwise, I honestly think it's
better to remove the stabilization period at the start, which benefits bwe and
similar more. 

[hbos_chromium, 2016/12/01 15:52:03]

Hmm. I don't have any bitrates at my disposal, but how about I run for 60
seconds, do bytesBefore = getStats and run for another 10 seconds and do
bytesAfter = getStats, then report the average bytes/second over that 10 second
period? This should be fairly stable, I would think.

+// Sums up "RTC[In/Out]boundRTPStreamStats.bytes_[received/sent]" values.
+double GetTotalRTPStreamBytes(
+    RTCStatsReportDictionary* report, bool inbound, const char* media_type) {
+  const char* type = inbound ? "inbound-rtp" : "outbound-rtp";
+  const char* bytes_name = inbound ? "bytesReceived" : "bytesSent";
+  double total_bytes = 0.0;
+  report->ForEach([&type, &bytes_name, &media_type, &total_bytes](
+      const RTCStatsDictionary& stats) {
+    if (stats.GetString("type") == type &&
+        stats.GetString("mediaType") == media_type) {
+      total_bytes += stats.GetNumber(bytes_name);
+    }
+  });
+  return total_bytes;
+}
+
+double GetAudioBytesSent(RTCStatsReportDictionary* report) {
+  return GetTotalRTPStreamBytes(report, false, "audio");
+}
+
+double GetAudioBytesReceived(RTCStatsReportDictionary* report) {
+  return GetTotalRTPStreamBytes(report, true, "audio");
+}
+
+double GetVideoBytesSent(RTCStatsReportDictionary* report) {
+  return GetTotalRTPStreamBytes(report, false, "video");
+}
+
+double GetVideoBytesReceived(RTCStatsReportDictionary* report) {
+  return GetTotalRTPStreamBytes(report, true, "video");
+}
+
+// This is different from |WebRtcPerfBrowserTest| in that it uses the
+// promise-based "RTCPeerConnection.getStats" (as opposed to
+// "chrome://webrtc-internals/"). The stats provided are different.
+class WebRtcPerformanceBrowserTest : public WebRtcTestBase {

[phoglund_chromium, 2016/12/01 15:42:30]

WebRtcPerfBrowserTests and WebRtcPerformanceBrowserTests? Umm, no. Rename the
old one to WebRtcInternalsPerfBrowserTests and this one to
WebRtcStatsPerfBrowserTest or something. You'll have to ask Edward to update the
--gtest_filter args in the bot config, so the WebRTC bots pick them up, though.

[hbos_chromium, 2016/12/02 14:55:46]

Yeah that's much better. Done.

+ public:
+  void SetUpInProcessBrowserTestFixture() override {
+    DetectErrorsInJavaScript();
+  }
+
+  void SetUpCommandLine(base::CommandLine* command_line) override {
+    // Ensure the infobar is enabled, since we expect that in this test.
+    EXPECT_FALSE(command_line->HasSwitch(switches::kUseFakeUIForMediaStream));
+
+    // Play a suitable, somewhat realistic video file.
+    base::FilePath input_video = test::GetReferenceFilesDir()
+        .Append(test::kReferenceFileName360p)
+        .AddExtension(test::kY4mFileExtension);
+    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
+                                   input_video);
+    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
+
+    command_line->AppendSwitchASCII(switches::kEnableBlinkFeatures,
+                                    "RTCPeerConnectionNewGetStats");
+  }
+
+  void RunsAudioAndVideoCallFor60Secs(
+      const std::string& audio_codec, const std::string& video_codec) {
+    ASSERT_TRUE(test::HasReferenceFilesInCheckout());
+    ASSERT_TRUE(embedded_test_server()->Start());
+
+    ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 100)
+        << "This is a long-running test; you must specify "
+           "--ui-test-action-max-timeout to have a value of at least 100000.";
+
+    content::WebContents* left_tab =
+        OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
+    content::WebContents* right_tab =
+        OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
+
+    SetupPeerconnectionWithLocalStream(left_tab);
+    SetupPeerconnectionWithLocalStream(right_tab);
+    SetDefaultAudioCodec(left_tab, audio_codec);
+    SetDefaultAudioCodec(right_tab, audio_codec);
+    SetDefaultVideoCodec(left_tab, video_codec);
+    SetDefaultVideoCodec(right_tab, video_codec);
+    NegotiateCall(left_tab, right_tab);
+    StartDetectingVideo(left_tab, "remote-view");
+    StartDetectingVideo(right_tab, "remote-view");
+    WaitForVideoToPlay(left_tab);
+    WaitForVideoToPlay(right_tab);
+
+    // Let call last for 60 seconds so that values may stabilize, bandwidth can
+    // ramp up, etc.
+    test::SleepInJavascript(left_tab, 60000);
+
+    scoped_refptr<RTCStatsReportDictionary> report =
+        GetStatsReportDictionary(left_tab);
+
+    if (audio_codec != kUseDefaultAudioCodec) {
+      double audio_bytes_sent = GetAudioBytesSent(report.get());
+      double audio_bytes_received = GetAudioBytesReceived(report.get());
+
+      std::string audio_codec_modifier = "_" + audio_codec;
+      perf_test::PrintResult(
+          "audio", audio_codec_modifier, "bytes_sent", audio_bytes_sent,
+          "bytes", false);
+      perf_test::PrintResult(
+          "audio", audio_codec_modifier, "bytes_received", audio_bytes_received,
+          "bytes", false);
+    }
+
+    if (video_codec != kUseDefaultVideoCodec) {
+      double video_bytes_sent = GetVideoBytesSent(report.get());
+      double video_bytes_received = GetVideoBytesReceived(report.get());
+
+      std::string video_codec_modifier = "_" + video_codec;
+      perf_test::PrintResult(
+          "video", video_codec_modifier, "bytes_sent", video_bytes_sent,
+          "bytes", false);
+      perf_test::PrintResult(
+          "video", video_codec_modifier, "bytes_received", video_bytes_received,
+          "bytes", false);
+    }
+
+    HangUp(left_tab);
+    HangUp(right_tab);
+  }
+};
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_AudioCodec_opus) {
+  RunsAudioAndVideoCallFor60Secs("opus", kUseDefaultVideoCodec);
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_AudioCodec_ISAC) {
+  RunsAudioAndVideoCallFor60Secs("ISAC", kUseDefaultVideoCodec);
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_AudioCodec_G722) {
+  RunsAudioAndVideoCallFor60Secs("G722", kUseDefaultVideoCodec);
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_AudioCodec_PCMU) {
+  RunsAudioAndVideoCallFor60Secs("PCMU", kUseDefaultVideoCodec);
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_AudioCodec_PCMA) {
+  RunsAudioAndVideoCallFor60Secs("PCMA", kUseDefaultVideoCodec);
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_VideoCodec_VP8) {
+  RunsAudioAndVideoCallFor60Secs(kUseDefaultAudioCodec, "VP8");
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_VideoCodec_VP9) {
+  RunsAudioAndVideoCallFor60Secs(kUseDefaultAudioCodec, "VP9");
+}
+
+#if BUILDFLAG(RTC_USE_H264)
+
+IN_PROC_BROWSER_TEST_F(WebRtcPerformanceBrowserTest,
+                       MANUAL_RunsAudioAndVideoCallFor60Secs_VideoCodec_H264) {
+  // Only run test if run-time feature corresponding to |rtc_use_h264| is on.
+  if (!base::FeatureList::IsEnabled(content::kWebRtcH264WithOpenH264FFmpeg)) {
+    LOG(WARNING) << "Run-time feature WebRTC-H264WithOpenH264FFmpeg disabled. "
+        "Skipping WebRtcPerfBrowserTest."
+        "MANUAL_RunsAudioAndVideoCallFor60Secs_VideoCodec_H264 (test \"OK\")";
+    return;
+  }
+  RunsAudioAndVideoCallFor60Secs(kUseDefaultAudioCodec, "H264");
+}
+
+#endif  // BUILDFLAG(RTC_USE_H264)
+
+}  // namespace
+
+}  // namespace content