Adding support for animated pages in the FrameRate tests

chrome/test/perf/frame_rate/frame_rate_tests.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <map>
 
 #include "base/file_util.h"
 #include "base/path_service.h"
 #include "base/string_number_conversions.h"
 #include "base/test/test_timeouts.h"
 #include "base/utf_string_conversions.h"
 #include "chrome/common/chrome_paths.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/test/automation/tab_proxy.h"
 #include "chrome/test/ui/javascript_test_util.h"
 #include "chrome/test/ui/ui_perf_test.h"
 #include "net/base/net_util.h"
+#include "ui/gfx/gl/gl_switches.h"
 
-namespace {
+// namespace must be named so that the PrintTo overload is externally visible
+namespace perf {
 
-class FrameRateTest : public UIPerfTest {
+enum FrameRateTestFlags {
+  kMakeBodyComposited = 1 << 0,
+  kDisableVsync       = 1 << 1,
+  kDisableGpu         = 1 << 2,
+  kUseReferenceBuild  = 1 << 3,
+};
+
+// Wrap flags into a class so that PrintTo can be overloaded
+class FrameRateTestVariant {
+ public:
+  explicit FrameRateTestVariant(unsigned flags) : flags_(flags) {}
+  unsigned GetFlags() const {return flags_;}
+ private:
+  unsigned flags_;
+};
+
+void PrintTo(const FrameRateTestVariant& v, std::ostream* os) {

[nduca, 2011/10/20 20:09:27]

Do we need this bit now? If we don't need it, I'd prefer we don't have it.

+  bool has_val = false;
+
+#define PRINTFLAG(flag_name) \
+  if (v.GetFlags() & flag_name) { \
+    if (has_val) { \
+      *os << " | "; \
+    } else { \
+      has_val = true; \
+    } \
+    *os << #flag_name; \
+  }
+
+  PRINTFLAG(kMakeBodyComposited)
+  PRINTFLAG(kDisableVsync)
+  PRINTFLAG(kDisableGpu)
+  PRINTFLAG(kUseReferenceBuild)
+
+  if (!has_val) {
+    *os << "0";
+  }
+}
+
+std::string ParamTestNameSuffix(FrameRateTestVariant param) {

[nduca, 2011/10/20 20:09:27]

Move this up to the flags? And make it take in the FrameRateTestFlags intead?
E.g. GetSuffixForTestFlags?

+  std::string suffix;
+  if (param.GetFlags() & kMakeBodyComposited)
+    suffix += "_comp";
+  if (param.GetFlags() & kDisableVsync)
+    suffix += "_novsync";
+  if (param.GetFlags() & kDisableGpu)
+    suffix += "_nogpu";
+  if (param.GetFlags() & kUseReferenceBuild)
+    suffix += "_ref";
+  return suffix;
+}
+
+class FrameRateTest
+  : public UIPerfTest
+  , public ::testing::WithParamInterface<FrameRateTestVariant> {

[nduca, 2011/10/20 20:09:27]

Can we parameterize tests on the enum directly? NBD if not, but it'd be cool if
we didn't need the FrameRateTestVariant.

  public:
   FrameRateTest() {
     show_window_ = true;
     dom_automation_enabled_ = true;
     // Since this is a performance test, try to use the host machine's GPU
     // instead of falling back to software-rendering.
     force_use_osmesa_ = false;
     disable_accelerated_compositing_ = false;
   }
 
   virtual FilePath GetDataPath(const std::string& name) {
     // Make sure the test data is checked out.
     FilePath test_path;
     PathService::Get(chrome::DIR_TEST_DATA, &test_path);
     test_path = test_path.Append(FILE_PATH_LITERAL("perf"));
     test_path = test_path.Append(FILE_PATH_LITERAL("frame_rate"));
     test_path = test_path.Append(FILE_PATH_LITERAL("content"));
     test_path = test_path.AppendASCII(name);
     return test_path;
   }
 
   virtual void SetUp() {
-    // UI tests boot up render views starting from about:blank. This causes the
-    // renderer to start up thinking it cannot use the GPU. To work around that,
-    // and allow the frame rate test to use the GPU, we must pass
-    // kAllowWebUICompositing.
+    if (GetParam().GetFlags() & kUseReferenceBuild) {
+      UseReferenceBuild();
+    }
+
+    // UI tests boot up render views starting from about:blank. This causes
+    // the renderer to start up thinking it cannot use the GPU. To work
+    // around that, and allow the frame rate test to use the GPU, we must
+    // pass kAllowWebUICompositing.
     launch_arguments_.AppendSwitch(switches::kAllowWebUICompositing);
 
+    if (GetParam().GetFlags() & kDisableGpu) {
+      launch_arguments_.AppendSwitch(switches::kDisableAcceleratedCompositing);
+      launch_arguments_.AppendSwitch(switches::kDisableExperimentalWebGL);
+    }
+
+    if (GetParam().GetFlags() & kDisableVsync) {
+      launch_arguments_.AppendSwitch(switches::kDisableGpuVsync);
+    }
+
     UIPerfTest::SetUp();
   }
 
-  void RunTest(const std::string& name,
-               const std::string& suffix,
-               bool make_body_composited) {
+  void RunTest(const std::string& name) {
     FilePath test_path = GetDataPath(name);
     ASSERT_TRUE(file_util::DirectoryExists(test_path))
         << "Missing test directory: " << test_path.value();
 
     test_path = test_path.Append(FILE_PATH_LITERAL("test.html"));
 
     scoped_refptr<TabProxy> tab(GetActiveTab());
     ASSERT_TRUE(tab.get());
 
     ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS,
               tab->NavigateToURL(net::FilePathToFileURL(test_path)));
 
-    if (make_body_composited) {
+    if (GetParam().GetFlags() & kMakeBodyComposited) {
       ASSERT_TRUE(tab->NavigateToURLAsync(
-          GURL("javascript:__make_body_composited();")));
+        GURL("javascript:__make_body_composited();")));
     }
 
+    // Block until initialization completes.
+    ASSERT_TRUE(WaitUntilJavaScriptCondition(
+        tab, L"", L"window.domAutomationController.send(__initialized);",
+        TestTimeouts::large_test_timeout_ms()));
+
     // Start the tests.
     ASSERT_TRUE(tab->NavigateToURLAsync(GURL("javascript:__start_all();")));
 
     // Block until the tests completes.
     ASSERT_TRUE(WaitUntilJavaScriptCondition(
         tab, L"", L"window.domAutomationController.send(!__running_all);",
         TestTimeouts::large_test_timeout_ms()));
 
     // Read out the results.
     std::wstring json;
     ASSERT_TRUE(tab->ExecuteAndExtractString(
         L"",
         L"window.domAutomationController.send("
         L"JSON.stringify(__calc_results_total()));",
         &json));
 
     std::map<std::string, std::string> results;
     ASSERT_TRUE(JsonDictionaryToMap(WideToUTF8(json), &results));
 
     ASSERT_TRUE(results.find("mean") != results.end());
     ASSERT_TRUE(results.find("sigma") != results.end());
     ASSERT_TRUE(results.find("gestures") != results.end());
     ASSERT_TRUE(results.find("means") != results.end());
     ASSERT_TRUE(results.find("sigmas") != results.end());
 
-    std::string trace_name = "fps" + suffix;
+    std::string trace_name = "fps";
+    trace_name += ParamTestNameSuffix(GetParam());
     printf("GESTURES %s: %s= [%s] [%s] [%s]\n", name.c_str(),
                                                 trace_name.c_str(),
                                                 results["gestures"].c_str(),
                                                 results["means"].c_str(),
                                                 results["sigmas"].c_str());
 
     std::string mean_and_error = results["mean"] + "," + results["sigma"];
     PrintResultMeanAndError(name, "", trace_name, mean_and_error,
                             "frames-per-second", true);
   }
 };
 
-class FrameRateTest_Reference : public FrameRateTest {
- public:
-  void SetUp() {
-    UseReferenceBuild();
-    FrameRateTest::SetUp();
-  }
-};
-
-#define FRAME_RATE_TEST(content) \
-TEST_F(FrameRateTest, content) { \
-  RunTest(#content, "", false); \
-} \
-TEST_F(FrameRateTest_Reference, content) { \
-  RunTest(#content, "_ref", false); \
-}
-
+FrameRateTestVariant kTestVariant_Plain(0);
+FrameRateTestVariant kTestVariant_Comp(kMakeBodyComposited);
+FrameRateTestVariant kTestVariant_Reference(kUseReferenceBuild);
+FrameRateTestVariant kTestVariant_Comp_Reference(kMakeBodyComposited
+                                                 | kUseReferenceBuild);
+FrameRateTestVariant kTestVariant_NoVsync(kDisableVsync);
+FrameRateTestVariant kTestVariant_NoGpu(kDisableGpu);
+FrameRateTestVariant kTestVariant_NoVsync_Reference(kDisableVsync
+                                                    | kUseReferenceBuild);
 
 // Tests that trigger compositing with a -webkit-translateZ(0)
 #define FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(content) \
-TEST_F(FrameRateTest, content) { \
-  RunTest(#content, "", false); \
-} \
-TEST_F(FrameRateTest, content ## _comp) { \
-  RunTest(#content, "_comp", true); \
-} \
-TEST_F(FrameRateTest_Reference, content) { \
-  RunTest(#content, "_ref", false); \
-} \
-TEST_F(FrameRateTest_Reference, content ## _comp) { \
-  RunTest(#content, "_comp_ref", true); \
+TEST_P(FrameRateTest, content) { \
+  RunTest(#content); \

[nduca, 2011/10/20 20:09:27]

typedef here? Its awkward how we use FrameRateTest here but FrameRateCanvasTest
down below. Ideally, they'd follow a similar pattern.

 }
 
 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(blank);
 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(googleblog);
 
+INSTANTIATE_TEST_CASE_P(, FrameRateTest, ::testing::Values(

[nduca, 2011/10/20 20:09:27]

can we put this after test_p and before
frame_rate_test_with_and_without-accelerated_compositing?

+                        kTestVariant_Plain,
+                        kTestVariant_Comp,
+                        kTestVariant_Reference,
+                        kTestVariant_Comp_Reference));
+
+// Must use a different class name to avoid test instantiation conflicts
+// with FrameRateTest (used above). An alias is good enough.
+typedef FrameRateTest FrameRateCanvasTest;
+
+#define FRAME_RATE_TEST_ANIMATED_2D_CANVAS(content) \
+TEST_P(FrameRateCanvasTest, content) { \
+  RunTest(#content); \
+} \
+
+INSTANTIATE_TEST_CASE_P(, FrameRateCanvasTest, ::testing::Values(

[nduca, 2011/10/20 20:09:27]

Lets file a bug for "don't run 5 tests for every single canvas frame rate test."
I don't want that bit of feedback to fall on the floor --- from a practical side
and from a bot-cycling-time perspective, this is a problem waiting to happen.

+                        kTestVariant_Plain,
+                        kTestVariant_NoVsync,
+                        kTestVariant_NoGpu,
+                        kTestVariant_Reference,
+                        kTestVariant_NoVsync_Reference));
+
 }  // namespace