Adding support for animated pages in the FrameRate tests

chrome/test/perf/frame_rate/frame_rate_tests.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <map>
 
 #include "base/file_util.h"
 #include "base/path_service.h"
 #include "base/string_number_conversions.h"
 #include "base/test/test_timeouts.h"
 #include "base/utf_string_conversions.h"
 #include "chrome/common/chrome_paths.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/test/automation/tab_proxy.h"
 #include "chrome/test/ui/javascript_test_util.h"
 #include "chrome/test/ui/ui_perf_test.h"
 #include "net/base/net_util.h"
 
 namespace {
 
-class FrameRateTest : public UIPerfTest {
+enum FrameRateTestFlags {
+  kMakeBodyComposited = 1 << 0,
+  kDisableRaf         = 1 << 1,
+  kDisableGpu         = 1 << 2,
+  kReference          = 1 << 3,
+};
+
+template<unsigned TestFlags>
+class FrameRateTest_ : public UIPerfTest {
  public:
-  FrameRateTest() {
+  FrameRateTest_() {
     show_window_ = true;
     dom_automation_enabled_ = true;
     // Since this is a performance test, try to use the host machine's GPU
     // instead of falling back to software-rendering.
     force_use_osmesa_ = false;
     disable_accelerated_compositing_ = false;
   }
 
   virtual FilePath GetDataPath(const std::string& name) {
     // Make sure the test data is checked out.
     FilePath test_path;
     PathService::Get(chrome::DIR_TEST_DATA, &test_path);
     test_path = test_path.Append(FILE_PATH_LITERAL("perf"));
     test_path = test_path.Append(FILE_PATH_LITERAL("frame_rate"));
     test_path = test_path.Append(FILE_PATH_LITERAL("content"));
     test_path = test_path.AppendASCII(name);
     return test_path;
   }
 
   virtual void SetUp() {
-    // UI tests boot up render views starting from about:blank. This causes the
-    // renderer to start up thinking it cannot use the GPU. To work around that,
-    // and allow the frame rate test to use the GPU, we must pass
-    // kAllowWebUICompositing.
-    launch_arguments_.AppendSwitch(switches::kAllowWebUICompositing);
+    if (TestFlags & kReference) {
+      UseReferenceBuild();
+    }
+
+    if (TestFlags & kDisableGpu) {

[nduca, 2011/10/14 21:01:46]

Do we have a global flag for this? @zmo might know, this feels fragile to have
to maintain (oops, someone added somethign else and now the frame rate
"disabled" tests are accidentally using the gpu"). If there isn't a central
flag, we shoudl add one or move this into test_launcher_utils.

+      launch_arguments_.AppendSwitch(switches::kDisableAccelerated2dCanvas);
+      launch_arguments_.AppendSwitch(switches::kDisableAcceleratedCompositing);
+      launch_arguments_.AppendSwitch(switches::kDisableAcceleratedLayers);
+      launch_arguments_.AppendSwitch(switches::kDisableAcceleratedPlugins);
+      launch_arguments_.AppendSwitch(switches::kDisableAcceleratedVideo);
+    } else {
+      // UI tests boot up render views starting from about:blank. This causes
+      // the renderer to start up thinking it cannot use the GPU. To work
+      // around that, and allow the frame rate test to use the GPU, we must
+      // pass kAllowWebUICompositing.
+      launch_arguments_.AppendSwitch(switches::kAllowWebUICompositing);

[nduca, 2011/10/14 21:01:46]

Append this always? There's no harm in it if we put it with the disable flags.

+    }
 
     UIPerfTest::SetUp();
   }
 
   void RunTest(const std::string& name,
-               const std::string& suffix,
-               bool make_body_composited) {
+               const std::string& suffix) {
     FilePath test_path = GetDataPath(name);
     ASSERT_TRUE(file_util::DirectoryExists(test_path))
         << "Missing test directory: " << test_path.value();
 
     test_path = test_path.Append(FILE_PATH_LITERAL("test.html"));
 
     scoped_refptr<TabProxy> tab(GetActiveTab());
     ASSERT_TRUE(tab.get());
 
     ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS,
               tab->NavigateToURL(net::FilePathToFileURL(test_path)));
 
-    if (make_body_composited) {
+    if (TestFlags & kMakeBodyComposited) {
       ASSERT_TRUE(tab->NavigateToURLAsync(
-          GURL("javascript:__make_body_composited();")));
+        GURL("javascript:__make_body_composited();")));
     }
 
+    if (TestFlags & kDisableRaf) {
+      ASSERT_TRUE(tab->NavigateToURLAsync(
+        GURL("javascript:__init_raf(false);")));
+    }
+
+    // Block until initialization completes.
+    ASSERT_TRUE(WaitUntilJavaScriptCondition(
+        tab, L"", L"window.domAutomationController.send(__initialized);",
+        TestTimeouts::large_test_timeout_ms()));
+
     // Start the tests.
     ASSERT_TRUE(tab->NavigateToURLAsync(GURL("javascript:__start_all();")));
 
     // Block until the tests completes.
     ASSERT_TRUE(WaitUntilJavaScriptCondition(
         tab, L"", L"window.domAutomationController.send(!__running_all);",
         TestTimeouts::large_test_timeout_ms()));
 
     // Read out the results.
     std::wstring json;
@@ skipping 18 matching lines @@
                                                 results["gestures"].c_str(),
                                                 results["means"].c_str(),
                                                 results["sigmas"].c_str());
 
     std::string mean_and_error = results["mean"] + "," + results["sigma"];
     PrintResultMeanAndError(name, "", trace_name, mean_and_error,
                             "frames-per-second", true);
   }
 };
 
-class FrameRateTest_Reference : public FrameRateTest {
- public:
-  void SetUp() {
-    UseReferenceBuild();
-    FrameRateTest::SetUp();
-  }
-};
+typedef FrameRateTest_<0>

[nduca, 2011/10/14 21:01:46]

I think gtest has parameterized tests, though it might not make sense here. Take
a peek and tell me if I'm full of it? :)

+  FrameRateTest;
+typedef FrameRateTest_<kReference>
+  FrameRateTest_Reference;
+typedef FrameRateTest_<kMakeBodyComposited>
+  FrameRateTest_Comp;
+typedef FrameRateTest_<kReference | kMakeBodyComposited>
+  FrameRateTest_Comp_Reference;
+typedef FrameRateTest_<kDisableRaf>
+  FrameRateTest_NoRaf;
+typedef FrameRateTest_<kReference | kDisableRaf>
+  FrameRateTest_NoRaf_Reference;
+typedef FrameRateTest_<kDisableGpu>
+  FrameRateTest_NoGpu;
+typedef FrameRateTest_<kReference | kDisableGpu>
+  FrameRateTest_NoGpu_Reference;
+typedef FrameRateTest_<kDisableRaf | kDisableGpu>
+  FrameRateTest_NoRaf_NoGpu;
+typedef FrameRateTest_<kReference | kDisableRaf | kDisableGpu>
+  FrameRateTest_NoRaf_NoGpu_Reference;
+
 
 #define FRAME_RATE_TEST(content) \
 TEST_F(FrameRateTest, content) { \
-  RunTest(#content, "", false); \
+  RunTest(#content, ""); \
 } \
 TEST_F(FrameRateTest_Reference, content) { \
-  RunTest(#content, "_ref", false); \
+  RunTest(#content, "_ref"); \
 }
 
 
 // Tests that trigger compositing with a -webkit-translateZ(0)
 #define FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(content) \
 TEST_F(FrameRateTest, content) { \
-  RunTest(#content, "", false); \
+  RunTest(#content, ""); \
 } \
-TEST_F(FrameRateTest, content ## _comp) { \
-  RunTest(#content, "_comp", true); \
+TEST_F(FrameRateTest_Comp, content ## _comp) { \
+  RunTest(#content, "_comp"); \
 } \
 TEST_F(FrameRateTest_Reference, content) { \
-  RunTest(#content, "_ref", false); \
+  RunTest(#content, "_ref"); \
 } \
-TEST_F(FrameRateTest_Reference, content ## _comp) { \
-  RunTest(#content, "_comp_ref", true); \
+TEST_F(FrameRateTest_Comp_Reference, content ## _comp) { \
+  RunTest(#content, "_comp_ref"); \
 }
 
+// Tests for pages that have animated 2d canvas content.
+// Tests run with and without RAF, and with and without GPU acceleration.
+#define FRAME_RATE_TEST_ANIMATED_2D_CANVAS(content) \

[nduca, 2011/10/14 21:01:46]

I really think you should factor this differently.

I realize you're just covering the bases here, but I think you should take a
step back and go "my end goal is to make sure that I can track performance gains
and regressions against performance.

Do you really need to run 8 signals for every piece of canvas content? 

Did you consider defining different types of test content and then have
configuraitons tailored for the types of things you want to learn from those
tests?



That is to say, the vast majority of canvas tests probabaly exercise canvas
performance. I imagine for those, you just want to see the unbounded performance
and also the performance were they on a vsync'd machine (e.g. did it run at
60fps or not).

You *could* track software mode against all of these, but honestly, since for
the majority of things you know its a win. So why not just make a few specific
targeted pieces of content where you know they're not good vs software, and make
a configuration for those tests that does software and gpu mode.

There are a small number of tests that want to test raf/nonraf performance.
Again, won't 1 or 2 pieces of content work here?

+TEST_F(FrameRateTest, content) { \
+  RunTest(#content, ""); \
+} \
+TEST_F(FrameRateTest_NoRaf, content) { \
+  RunTest(#content, "_noraf"); \
+} \
+TEST_F(FrameRateTest_NoGpu, content) { \
+  RunTest(#content, "_nogpu"); \
+} \
+TEST_F(FrameRateTest_NoRaf_NoGpu, content) { \
+  RunTest(#content, "_noraf_nogpu"); \
+} \
+TEST_F(FrameRateTest_Reference, content) { \
+  RunTest(#content, "_ref"); \
+} \
+TEST_F(FrameRateTest_NoRaf_Reference, content) { \
+  RunTest(#content, "_noraf_ref"); \
+} \
+TEST_F(FrameRateTest_NoGpu_Reference, content) { \
+  RunTest(#content, "_nogpu_ref"); \
+} \
+TEST_F(FrameRateTest_NoRaf_NoGpu_Reference, content) { \
+  RunTest(#content, "_noraf_nogpu_ref"); \
+} \
+
 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(blank);

[nduca, 2011/10/14 21:01:46]

move these up next to the define?

 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(googleblog);
 
 }  // namespace