PPAPI/NaCl: Make related tests run in 1 fixture

ppapi/tests/test_case.h

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef PPAPI_TESTS_TEST_CASE_H_
 #define PPAPI_TESTS_TEST_CASE_H_
 
 #include <cmath>
 #include <limits>
+#include <map>
 #include <set>
 #include <string>
 
+#include "ppapi/c/dev/ppb_testing_dev.h"
 #include "ppapi/c/pp_resource.h"
-#include "ppapi/c/dev/ppb_testing_dev.h"
+#include "ppapi/c/pp_time.h"
 #include "ppapi/cpp/dev/scrollbar_dev.h"
 #include "ppapi/cpp/message_loop.h"
 #include "ppapi/cpp/view.h"
 #include "ppapi/tests/test_utils.h"
 #include "ppapi/tests/testing_instance.h"
 
 #if (defined __native_client__)
 #include "ppapi/cpp/var.h"
 #else
 #include "ppapi/cpp/private/var_private.h"
@@ skipping 11 matching lines @@
 class TestCase {
  public:
   explicit TestCase(TestingInstance* instance);
   virtual ~TestCase();
 
   // Optionally override to do testcase specific initialization.
   // Default implementation just returns true.
   virtual bool Init();
 
   // Override to implement the test case. It will be called after the plugin is
-  // first displayed, passing a string. If the string is empty, the
-  // should run all tests for this test case. Otherwise, it should run the test
-  // whose name matches test_filter exactly (if there is one). This should
-  // generally be implemented using the RUN_TEST* macros.
+  // first displayed, passing a string. If the string is empty, RunTests should
+  // run all tests for this test case. Otherwise, it must be a comma-delimited
+  // list of test names, possibly prefixed. E.g.:
+  //   "Foo_GoodTest,DISABLED_Foo_BadTest,Foo_OtherGoodTest"
+  // All listed tests which are not prefixed will be run.
+  //
+  // This should generally be implemented in a TestCase subclass using the
+  // RUN_TEST* macros.
   virtual void RunTests(const std::string& test_filter) = 0;
 
   static std::string MakeFailureMessage(const char* file, int line,
                                         const char* cmd);
 
 #if !(defined __native_client__)
   // Returns the scriptable test object for the current test, if any.
   // Internally, this uses CreateTestObject which each test overrides.
   pp::VarPrivate GetTestObject();
 #endif
@@ skipping 15 matching lines @@
   virtual bool HandleInputEvent(const pp::InputEvent& event);
 
   void IgnoreLeakedVar(int64_t id);
 
   TestingInstance* instance() { return instance_; }
 
   const PPB_Testing_Dev* testing_interface() { return testing_interface_; }
 
   static void QuitMainMessageLoop(PP_Instance instance);
 
+  const std::map<std::string, bool>& remaining_tests() {
+    return remaining_tests_;
+  }
+  const std::set<std::string>& skipped_tests() {
+    return skipped_tests_;
+  }
+
  protected:
 #if !(defined __native_client__)
   // Overridden by each test to supply a ScriptableObject corresponding to the
   // test. There can only be one object created for all tests in a given class,
   // so be sure your object is designed to be re-used.
   //
   // This object should be created on the heap. Ownership will be passed to the
   // caller. Return NULL if there is no supported test object (the default).
   virtual pp::deprecated::ScriptableObject* CreateTestObject();
 #endif
 
   // Checks whether the testing interface is available. Returns true if it is,
   // false otherwise. If it is not available, adds a descriptive error. This is
   // for use by tests that require the testing interface.
   bool CheckTestingInterface();
 
   // Makes sure the test is run over HTTP.
   bool EnsureRunningOverHTTP();
 
+  // Returns true if |filter| only contains a TestCase name, which normally
+  // means "run all tests". Some TestCases require special setup for individual
+  // tests, and can use this function to decide whether to ignore those tests.
+  bool ShouldRunAllTests(const std::string& filter);
+
   // Return true if the given test name matches the filter. This is true if
-  // (a) filter is empty or (b) test_name and filter match exactly.
-  bool MatchesFilter(const std::string& test_name, const std::string& filter);
+  // (a) filter is empty or (b) test_name matches a test name listed in filter
+  // exactly.
+  bool ShouldRunTest(const std::string& test_name, const std::string& filter);
 
   // Check for leaked resources and vars at the end of the test. If any exist,
   // return a string with some information about the error. Otherwise, return
   // an empty string.
   //
   // You should pass the error string from the test so far; if it is non-empty,
   // CheckResourcesAndVars will do nothing and return the same string.
   std::string CheckResourcesAndVars(std::string errors);
 
+  PP_TimeTicks NowInTimeTicks();
+
   // Run the given test method on a background thread and return the result.
   template <class T>
   std::string RunOnThread(std::string(T::*test_to_run)()) {
 #ifdef ENABLE_PEPPER_THREADING
     if (!testing_interface_) {
       return "Testing blocking callbacks requires the testing interface. In "
              "Chrome, use the --enable-pepper-testing flag.";
     }
     // These tests are only valid if running out-of-process (threading is not
     // supported in-process). For in-process, just consider it a pass.
@@ skipping 79 matching lines @@
 
   static void DoQuitMainMessageLoop(void* pp_instance, int32_t result);
 
   // Passed when creating completion callbacks in some tests. This determines
   // what kind of callback we use for the test.
   CallbackType callback_type_;
 
   // Var ids that should be ignored when checking for leaks on shutdown.
   std::set<int64_t> ignored_leaked_vars_;
 
+  // The tests that were found in test_filter but have not yet been run. The
+  // bool indicates whether the test should be run (i.e., it will be false if
+  // the test name was prefixed in the test_filter string).
+  //
+  // This is initialized lazily the first time that ShouldRunTest is called by
+  // RunTests. When RunTests is finished, this should be empty. Any remaining
+  // tests are tests that were listed in the test_filter but didn't match
+  // any calls to ShouldRunTest, meaning it was probably a typo. TestingInstance
+  // should log this and consider it a failure.
+  std::map<std::string, bool> remaining_tests_;
+  // Flag indicating whether we have populated remaining_tests_ yet.
+  bool have_populated_remaining_tests_;
+
+  // If ShouldRunTest is called but the given test name doesn't match anything
+  // in the test_filter, the test name will be added here. This allows
+  // TestingInstance to detect when not all tests were listed.
+  std::set<std::string> skipped_tests_;
+
 #if !(defined __native_client__)
   // Holds the test object, if any was retrieved from CreateTestObject.
   pp::VarPrivate test_object_;
 #endif
 };
 
 // This class is an implementation detail.
 class TestCaseFactory {
  public:
   typedef TestCase* (*Method)(TestingInstance* instance);
@@ skipping 28 matching lines @@
     return new Test##name(instance);                                        \
   }                                                                         \
   static TestCaseFactory g_Test##name_factory(                              \
     #name, &Test##name##_FactoryMethod                                      \
   )
 
 // Helper macro for calling functions implementing specific tests in the
 // RunTest function. This assumes the function name is TestFoo where Foo is the
 // test |name|.
 #define RUN_TEST(name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_OPTIONAL); \
-    instance_->LogTest(#name, CheckResourcesAndVars(Test##name())); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest(#name, \
+                       CheckResourcesAndVars(Test##name()), \
+                       start_time); \
   }
 
 // Like RUN_TEST above but forces functions taking callbacks to complete
 // asynchronously on success or error.
 #define RUN_TEST_FORCEASYNC(name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_REQUIRED); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
     instance_->LogTest(#name"ForceAsync", \
-                       CheckResourcesAndVars(Test##name())); \
+                       CheckResourcesAndVars(Test##name()), \
+                       start_time); \
   }
 
 #define RUN_TEST_BLOCKING(test_case, name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_BLOCKING); \
-    instance_->LogTest(#name"Blocking", \
-        CheckResourcesAndVars(RunOnThread(&test_case::Test##name))); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest( \
+        #name"Blocking", \
+        CheckResourcesAndVars(RunOnThread(&test_case::Test##name)), \
+        start_time); \
   }
 
 #define RUN_TEST_BACKGROUND(test_case, name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
-    instance_->LogTest(#name"Background", \
-        CheckResourcesAndVars(RunOnThread(&test_case::Test##name))); \
+  if (ShouldRunTest(#name, test_filter)) { \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest( \
+        #name"Background", \
+        CheckResourcesAndVars(RunOnThread(&test_case::Test##name)), \
+        start_time); \
   }
 
 #define RUN_TEST_FORCEASYNC_AND_NOT(name, test_filter) \
   do { \
     RUN_TEST_FORCEASYNC(name, test_filter); \
     RUN_TEST(name, test_filter); \
   } while (false)
 
 // Run a test with all possible callback types.
 #define RUN_CALLBACK_TEST(test_case, name, test_filter) \
   do { \
     RUN_TEST_FORCEASYNC(name, test_filter); \
     RUN_TEST(name, test_filter); \
     RUN_TEST_BLOCKING(test_case, name, test_filter); \
     RUN_TEST_BACKGROUND(test_case, name, test_filter); \
   } while (false)
 
 #define RUN_TEST_WITH_REFERENCE_CHECK(name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_OPTIONAL); \
     uint32_t objects = testing_interface_->GetLiveObjectsForInstance( \
         instance_->pp_instance()); \
     std::string error_message = Test##name(); \
     if (error_message.empty() && \
         testing_interface_->GetLiveObjectsForInstance( \
             instance_->pp_instance()) != objects) \
       error_message = MakeFailureMessage(__FILE__, __LINE__, \
           "reference leak check"); \
-    instance_->LogTest(#name, error_message); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest(#name, \
+                       error_message, \
+                       start_time); \
   }
 // TODO(dmichael): Add CheckResourcesAndVars above when Windows tests pass
 //                 cleanly. crbug.com/173503
 
 // Helper macros for checking values in tests, and returning a location
 // description of the test fails.
 #define ASSERT_TRUE(cmd) \
   if (!(cmd)) { \
     return MakeFailureMessage(__FILE__, __LINE__, #cmd); \
   }
 #define ASSERT_FALSE(cmd) ASSERT_TRUE(!(cmd))
 #define ASSERT_EQ(a, b) ASSERT_TRUE((a) == (b))
 #define ASSERT_NE(a, b) ASSERT_TRUE((a) != (b))
 
 #define ASSERT_DOUBLE_EQ(a, b) ASSERT_TRUE( \
     std::fabs((a)-(b)) <= std::numeric_limits<double>::epsilon())
 
 // Runs |function| as a subtest and asserts that it has passed.
 #define ASSERT_SUBTEST_SUCCESS(function) \
   do { \
     std::string result = (function); \
     if (!result.empty()) \
       return result; \
   } while (false)
 
 #define PASS() return std::string()
 
 #endif  // PPAPI_TESTS_TEST_CASE_H_