add "did it run on GPU" check to frame_rate_tests.cc using AutomationProxy tracing feature

chrome/test/perf/frame_rate/frame_rate_tests.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <map>
 
 #include "base/command_line.h"
 #include "base/file_util.h"
 #include "base/path_service.h"
 #include "base/string_number_conversions.h"
 #include "base/test/test_timeouts.h"
+#include "base/test/trace_event_analyzer.h"
 #include "base/utf_string_conversions.h"
 #include "chrome/common/chrome_paths.h"
 #include "chrome/common/chrome_switches.h"
+#include "chrome/test/automation/automation_proxy.h"
 #include "chrome/test/automation/tab_proxy.h"
 #include "chrome/test/ui/javascript_test_util.h"
 #include "chrome/test/ui/ui_perf_test.h"
 #include "net/base/net_util.h"
+#include "ui/gfx/gl/gl_implementation.h"
 #include "ui/gfx/gl/gl_switches.h"
 
 namespace {
 
 enum FrameRateTestFlags {
   kRequiresGpu        = 1 << 0, // only execute test if --enable-gpu
   kDisableGpu         = 1 << 1, // run test without gpu acceleration
   kMakeBodyComposited = 1 << 2, // force the test to use the compositor
   kDisableVsync       = 1 << 3, // do not lock animation on vertical refresh
   kUseReferenceBuild  = 1 << 4, // run test using the reference chrome build
   kInternal           = 1 << 5, // Test uses internal test data
   kHasRedirect        = 1 << 6, // Test page contains an HTML redirect
 };
 
-std::string GetSuffixForTestFlags(int flags) {
+std::string GetSuffixForTestFlags(int flags, bool did_run_on_gpu) {
   std::string suffix;
   if (flags & kMakeBodyComposited)
     suffix += "_comp";
   if (flags & kDisableVsync)
     suffix += "_novsync";
   if (flags & kDisableGpu)
     suffix += "_nogpu";

[nduca, 2011/11/08 00:27:42]

does the _nogpu get affected? Can we reconcile the stuff @junov is trying to
achieve with this? Maybe ping him and if you two get to closure I'm happy.

[jbates, 2011/11/09 02:14:46]

Done.

   if (flags & kUseReferenceBuild)
     suffix += "_ref";
+  if (did_run_on_gpu)
+    suffix += "_gpu";
   return suffix;
 }
 
 class FrameRateTest
   : public UIPerfTest
   , public ::testing::WithParamInterface<int> {
  public:
   FrameRateTest() {
     show_window_ = true;
     dom_automation_enabled_ = true;
   }
 
+  virtual bool IsReferenceBuild() const {
+    return (GetParam() & kUseReferenceBuild);
+  }
+
   virtual FilePath GetDataPath(const std::string& name) {
     // Make sure the test data is checked out.
     FilePath test_path;
     PathService::Get(chrome::DIR_TEST_DATA, &test_path);
     test_path = test_path.Append(FILE_PATH_LITERAL("perf"));
     test_path = test_path.Append(FILE_PATH_LITERAL("frame_rate"));
     if (GetParam() & kInternal) {
       test_path = test_path.Append(FILE_PATH_LITERAL("private"));
     } else {
       test_path = test_path.Append(FILE_PATH_LITERAL("content"));
     }
     test_path = test_path.AppendASCII(name);
     return test_path;
   }
 
   virtual void SetUp() {
-    if (GetParam() & kUseReferenceBuild) {
+    if (IsReferenceBuild()) {
       UseReferenceBuild();
     }
 
     // UI tests boot up render views starting from about:blank. This causes
     // the renderer to start up thinking it cannot use the GPU. To work
     // around that, and allow the frame rate test to use the GPU, we must
     // pass kAllowWebUICompositing.
     launch_arguments_.AppendSwitch(switches::kAllowWebUICompositing);
 
     // Some of the tests may launch http requests through JSON or AJAX
@@ skipping 12 matching lines @@
       launch_arguments_.AppendSwitch(switches::kEnableAccelerated2dCanvas);
     }
 
     if (GetParam() & kDisableVsync) {
       launch_arguments_.AppendSwitch(switches::kDisableGpuVsync);
     }
 
     UIPerfTest::SetUp();
   }
 
+  bool DidRunOnGpu(const std::string& json_events) {
+    using namespace trace_analyzer;
+
+    // TODO(jbates): remove this check when ref builds are updated.
+    if (IsReferenceBuild())
+      return false;
+
+    // Check trace for GPU accleration.
+    scoped_ptr<TraceAnalyzer> analyzer(TraceAnalyzer::Create(json_events));
+    // Verify that sanity-check DoDeferredUpdate event exists.
+    EXPECT_TRUE(analyzer->FindOneEvent(Query(EVENT_NAME) ==
+                                       Query::String("DoDeferredUpdate")));
+
+    gfx::GLImplementation gl_impl = gfx::kGLImplementationNone;
+    const TraceEvent* gpu_event = analyzer->FindOneEvent(
+        Query(EVENT_NAME) == Query::String("SwapBuffers") &&
+        Query(EVENT_HAS_NUMBER_ARG, "GLImpl"));
+    if (gpu_event)
+      gl_impl = static_cast<gfx::GLImplementation>(
+          gpu_event->GetKnownArgAsInt("GLImpl"));
+    return (gl_impl == gfx::kGLImplementationDesktopGL ||
+            gl_impl == gfx::kGLImplementationEGLGLES2);
+  }
+
   void RunTest(const std::string& name) {
     if ((GetParam() & kRequiresGpu) &&
         !CommandLine::ForCurrentProcess()->HasSwitch("enable-gpu")) {
       printf("Test skipped: requires gpu\n");
       return;
     }
     FilePath test_path = GetDataPath(name);
     ASSERT_TRUE(file_util::DirectoryExists(test_path))
         << "Missing test directory: " << test_path.value();
 
@@ skipping 25 matching lines @@
     // that flag kHasRedirect is enabled for the current test.
     ASSERT_TRUE(WaitUntilJavaScriptCondition(
       tab, L"", L"window.domAutomationController.send(__initialized);",
       TestTimeouts::large_test_timeout_ms()));
 
     if (GetParam() & kMakeBodyComposited) {
       ASSERT_TRUE(tab->NavigateToURLAsync(
         GURL("javascript:__make_body_composited();")));
     }
 
+    // TODO(jbates): remove this check when ref builds are updated.
+    if (!IsReferenceBuild())
+      ASSERT_TRUE(automation()->BeginTracing("test_frame_rate_tests"));
+
     // Start the tests.
     ASSERT_TRUE(tab->NavigateToURLAsync(GURL("javascript:__start_all();")));
 
     // Block until the tests completes.
     ASSERT_TRUE(WaitUntilJavaScriptCondition(
         tab, L"", L"window.domAutomationController.send(!__running_all);",
         TestTimeouts::large_test_timeout_ms()));
 
+    // TODO(jbates): remove this check when ref builds are updated.
+    std::string json_events;
+    if (!IsReferenceBuild())
+      ASSERT_TRUE(automation()->EndTracing(&json_events));
+
     // Read out the results.
     std::wstring json;
     ASSERT_TRUE(tab->ExecuteAndExtractString(
         L"",
         L"window.domAutomationController.send("
         L"JSON.stringify(__calc_results_total()));",
         &json));
 
     std::map<std::string, std::string> results;
     ASSERT_TRUE(JsonDictionaryToMap(WideToUTF8(json), &results));
 
     ASSERT_TRUE(results.find("mean") != results.end());
     ASSERT_TRUE(results.find("sigma") != results.end());
     ASSERT_TRUE(results.find("gestures") != results.end());
     ASSERT_TRUE(results.find("means") != results.end());
     ASSERT_TRUE(results.find("sigmas") != results.end());
 
     std::string trace_name = "fps";
-    trace_name += GetSuffixForTestFlags(GetParam());
+    trace_name += GetSuffixForTestFlags(GetParam(), DidRunOnGpu(json_events));

[Justin Novosad, 2011/11/08 14:58:43]

Currently, the set of tests that run on the gpu are the tests that are run as
part of the gpu_frame_rate test (the gpu waterfall counterpart of the frame_rate
test) and do not have the kDisableGpu flag set.  I think that rather than adding
redundant info to the name, we could instead have a check.  something like:

bool ranOnGpu = DidRunOnGpu(json_events));
if (CommandLine::ForCurrentProcess()->HasSwitch("enable-gpu") && !(GetParam() &
kDisableGpu)) {
  ASSERT_TRUE(ranOnGpu);
} else {
  ASSERT_FALSE(ranOnGpu);
}

This way we would get a failure if a test that is expected to run on the GPU
ends up not running on the GPU, and vice versa.  I think it will be very useful
to validate that tests do not run on the GPU when they are not supposed to, like
on the perf waterfall. There is a risk that code churn in the command line
switches could cause the tests to be executed using mesa, which is not the
purpose of the test.

[jbates, 2011/11/09 02:14:46]

Done as discussed.

     printf("GESTURES %s: %s= [%s] [%s] [%s]\n", name.c_str(),
                                                 trace_name.c_str(),
                                                 results["gestures"].c_str(),
                                                 results["means"].c_str(),
                                                 results["sigmas"].c_str());
 
     std::string mean_and_error = results["mean"] + "," + results["sigma"];
     PrintResultMeanAndError(name, "", trace_name, mean_and_error,
                             "frames-per-second", true);
   }
@@ skipping 58 matching lines @@
                         kInternal | kHasRedirect | kUseReferenceBuild |
                           kRequiresGpu,
                         kInternal | kHasRedirect | kDisableVsync |
                           kRequiresGpu | kUseReferenceBuild));
 
 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(fireflies)
 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(FishIE)
 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(speedreading)
 
 }  // namespace