PPAPI/NaCl: Make related tests run in 1 fixture

ppapi/tests/testing_instance.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "ppapi/tests/testing_instance.h"
 
 #include <algorithm>
 #include <cstring>
+#include <iomanip>
 #include <sstream>
 #include <vector>
 
+#include "ppapi/cpp/core.h"
 #include "ppapi/cpp/module.h"
 #include "ppapi/cpp/var.h"
 #include "ppapi/cpp/view.h"
 #include "ppapi/tests/test_case.h"
 
 TestCaseFactory* TestCaseFactory::head_ = NULL;
 
 // Cookie value we use to signal "we're still working." See the comment above
 // the class declaration for how this works.
 static const char kProgressSignal[] = "...";
@@ skipping 37 matching lines @@
     } else if (std::strcmp(argn[i], "ssl_server_port") == 0) {
       ssl_server_port_ = atoi(argv[i]);
     }
   }
   // Create the proper test case from the argument.
   for (uint32_t i = 0; i < argc; i++) {
     if (std::strcmp(argn[i], "testcase") == 0) {
       if (argv[i][0] == '\0')
         break;
       current_case_ = CaseForTestName(argv[i]);
-      test_filter_ = FilterForTestName(argv[i]);
+      test_filter_ = argv[i];
       if (!current_case_)
         errors_.append(std::string("Unknown test case ") + argv[i]);
       else if (!current_case_->Init())
         errors_.append(" Test case could not initialize.");
       return true;
     }
   }
 
   // In DidChangeView, we'll dump out a list of all available tests.
   return true;
@@ skipping 33 matching lines @@
 void TestingInstance::EvalScript(const std::string& script) {
   SendTestCommand("EvalScript", script);
 }
 
 void TestingInstance::SetCookie(const std::string& name,
                                 const std::string& value) {
   SendTestCommand("SetCookie", name + "=" + value);
 }
 
 void TestingInstance::LogTest(const std::string& test_name,
-                              const std::string& error_message) {
+                              const std::string& error_message,
+                              PP_TimeTicks start_time) {
+  // Compute the time to run the test and save it in a string for logging:
+  PP_TimeTicks end_time(pp::Module::Get()->core()->GetTimeTicks());
+  std::ostringstream number_stream;
+  PP_TimeTicks elapsed_time(end_time - start_time);
+  number_stream << std::fixed << std::setprecision(3) << elapsed_time;
+  std::string time_string(number_stream.str());
+
   // Tell the browser we're still working.
   ReportProgress(kProgressSignal);
 
   number_tests_executed_++;
 
   std::string html;
   html.append("<div class=\"test_line\"><span class=\"test_name\">");
   html.append(test_name);
   html.append("</span> ");
   if (error_message.empty()) {
     html.append("<span class=\"pass\">PASS</span>");
   } else {
     html.append("<span class=\"fail\">FAIL</span>: <span class=\"err_msg\">");
     html.append(error_message);
     html.append("</span>");
 
     if (!errors_.empty())
       errors_.append(", ");  // Separator for different error messages.
     errors_.append(test_name + " FAIL: " + error_message);
   }
+  html.append(" <span class=\"time\">(");
+  html.append(time_string);
+  html.append("s)</span>");
+
   html.append("</div>");
   LogHTML(html);
 }
 
 void TestingInstance::AppendError(const std::string& message) {
   if (!errors_.empty())
     errors_.append(", ");
   errors_.append(message);
 }
 
@@ skipping 11 matching lines @@
     LogAvailableTests();
     errors_.append("FAIL: Only listed tests");
   } else {
     current_case_->RunTests(test_filter_);
 
     if (number_tests_executed_ == 0) {
       errors_.append("No tests executed. The test filter might be too "
                      "restrictive: '" + test_filter_ + "'.");
       LogError(errors_);
     }
-    else {
-      // Automated PyAuto tests rely on finding the exact strings below.
-      LogHTML(errors_.empty() ?
-              "<span class=\"pass\">[SHUTDOWN]</span> All tests passed." :
-              "<span class=\"fail\">[SHUTDOWN]</span> Some tests failed.");
+    if (current_case_->skipped_tests().size()) {
+      // TODO(dmichael): Convert all TestCases to run all tests in one fixture,
+      //                 and enable this check. Currently, a lot of our tests
+      //                 run 1 test per fixture, which is slow.
+      /*
+      errors_.append("Some tests were not listed and thus were not run. Make "
+                     "sure all tests are passed in the test_case URL (even if "
+                     "they are marked DISABLED_). Forgotten tests: ");
+      std::set<std::string>::const_iterator iter =
+          current_case_->skipped_tests().begin();
+      for (; iter != current_case_->skipped_tests().end(); ++iter) {
+        errors_.append(*iter);
+        errors_.append(" ");
+      }
+      LogError(errors_);
+      */
+    }
+    if (current_case_->remaining_tests().size()) {
+      errors_.append("Some listed tests were not found in the TestCase. Check "
+                     "the test names that were passed to make sure they match "
+                     "tests in the TestCase. Unknown tests: ");
+      std::map<std::string, bool>::const_iterator iter =
+          current_case_->remaining_tests().begin();
+      for (; iter != current_case_->remaining_tests().end(); ++iter) {
+        errors_.append(iter->first);
+        errors_.append(" ");
+      }
+      LogError(errors_);
     }
   }
 
   // Declare we're done by setting a cookie to either "PASS" or the errors.
   ReportProgress(errors_.empty() ? "PASS" : errors_);
   if (remove_plugin_)
     SendTestCommand("DidExecuteTests");
   // Note, DidExecuteTests unloads the plugin. We can't really do anthing after
   // this point.
 }
 
 TestCase* TestingInstance::CaseForTestName(const std::string& name) {
   std::string case_name = name.substr(0, name.find_first_of('_'));
   TestCaseFactory* iter = TestCaseFactory::head_;
   while (iter != NULL) {
     if (case_name == iter->name_)
       return iter->method_(this);
     iter = iter->next_;
   }
   return NULL;
 }
 
-std::string TestingInstance::FilterForTestName(const std::string& name) {
-  size_t delim = name.find_first_of('_');
-  if (delim != std::string::npos)
-    return name.substr(delim+1);
-  return "";
-}
-
 void TestingInstance::SendTestCommand(const std::string& command) {
   std::string msg("TESTING_MESSAGE:");
   msg += command;
   PostMessage(pp::Var(msg));
 }
 
 void TestingInstance::SendTestCommand(const std::string& command,
                                       const std::string& params) {
   SendTestCommand(command + ":" + params);
 }
@@ skipping 60 matching lines @@
   }
 };
 
 namespace pp {
 
 Module* CreateModule() {
   return new ::Module();
 }
 
 }  // namespace pp