Roll out framework change & update gpu tests to conform to the API changes

content/test/gpu/gpu_tests/gpu_integration_test_unittest.py

Index: content/test/gpu/gpu_tests/gpu_integration_test_unittest.py
diff --git a/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py b/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py
index 49462559374651b17e4cfad87a852e783c95c6a5..66b3ec5d59f6f9f618ee024cdff4d192d06a47a5 100644
--- a/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py
+++ b/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py
@@ -4,8 +4,8 @@
 
 import json
 import logging
-import mock
 import os
+import sys
 import tempfile
 import unittest
 
@@ -31,7 +31,7 @@ class SimpleIntegrationUnittest(gpu_integration_test.GpuIntegrationTest):
     super(SimpleIntegrationUnittest, self).setUp()
 
   @classmethod
-  def setUpClass(cls):
+  def SetUpProcess(cls):
     finder_options = fakes.CreateBrowserFinderOptions()
     finder_options.browser_options.platform = fakes.FakeLinuxPlatform()
     finder_options.output_formats = ['none']
@@ -85,7 +85,7 @@ class BrowserStartFailureIntegrationUnittest(
   _num_browser_starts = 0
 
   @classmethod
-  def setUpClass(cls):
+  def SetUpProcess(cls):
     cls._fake_browser_options = \
         fakes.CreateBrowserFinderOptions(execute_on_startup=cls.CrashOnStart)
     cls._fake_browser_options.browser_options.platform = \
@@ -135,7 +135,7 @@ class BrowserCrashAfterStartIntegrationUnittest(
   _num_browser_starts = 0
 
   @classmethod
-  def setUpClass(cls):
+  def SetUpProcess(cls):
     cls._fake_browser_options = fakes.CreateBrowserFinderOptions(
       execute_after_browser_creation=cls.CrashAfterStart)
     cls._fake_browser_options.browser_options.platform = \
@@ -185,8 +185,7 @@ class BrowserCrashAfterStartIntegrationUnittest(
 
 
 class GpuIntegrationTestUnittest(unittest.TestCase):
-  @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
-  def testSimpleIntegrationUnittest(self, mockInitDependencyManager):
+  def testSimpleIntegrationUnittest(self):
     self._RunIntegrationTest(
       'simple_integration_unittest', [
         'unexpected_error',
@@ -199,9 +198,7 @@ class GpuIntegrationTestUnittest(unittest.TestCase):
     # with these browser restarts, but this is at least a start.
     self.assertEquals(SimpleIntegrationUnittest._num_browser_starts, 6)
 
-  @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
-  def testIntegrationUnittestWithBrowserFailure(
-      self, mockInitDependencyManager):
+  def testIntegrationUnittestWithBrowserFailure(self):
     self._RunIntegrationTest(
       'browser_start_failure_integration_unittest', [], ['restart'])
     self.assertEquals( \
@@ -209,9 +206,7 @@ class GpuIntegrationTestUnittest(unittest.TestCase):
     self.assertEquals( \
       BrowserStartFailureIntegrationUnittest._num_browser_starts, 3)
 
-  @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
-  def testIntegrationUnittestWithBrowserCrashUponStart(
-      self, mockInitDependencyManager):
+  def testIntegrationUnittestWithBrowserCrashUponStart(self):
     self._RunIntegrationTest(
       'browser_crash_after_start_integration_unittest', [], ['restart'])
     self.assertEquals( \
@@ -220,23 +215,53 @@ class GpuIntegrationTestUnittest(unittest.TestCase):
       BrowserCrashAfterStartIntegrationUnittest._num_browser_starts, 3)
 
   def _RunIntegrationTest(self, test_name, failures, successes):
-    options = browser_test_runner.TestRunOptions()
-    # Suppress printing out information for passing tests.
-    options.verbosity = 0
     config = gpu_project_config.CONFIG
     temp_file = tempfile.NamedTemporaryFile(delete=False)
     temp_file.close()
     temp_file_name = temp_file.name
     try:
       browser_test_runner.Run(
-          config, options,
+          config,
           [test_name,
-           '--write-abbreviated-json-results-to=%s' % temp_file_name])
+           '--write-full-results-to=%s' % temp_file_name])
       with open(temp_file_name) as f:
         test_result = json.load(f)
-      self.assertEquals(test_result['failures'], failures)
-      self.assertEquals(test_result['successes'], successes)
-      self.assertEquals(test_result['valid'], True)
+      actual_successes, actual_failures = self._ExtracTestResults(test_result)
+      self.assertEquals(actual_failures, failures)
+      self.assertEquals(actual_successes, successes)
 
     finally:
       os.remove(temp_file_name)
+
+  def _ExtracTestResults(self, test_result):
+    delimiter = test_result['path_delimiter']
+    failures = []
+    successes = []
+    def _IsLeafNode(node):
+      test_dict = node[1]
+      return ('expected' in test_dict and
+              isinstance(test_dict['expected'], basestring))
+    node_queues = []
+    for t in test_result['tests']:
+      node_queues.append((t, test_result['tests'][t]))
+    while node_queues:
+      node = node_queues.pop()
+      full_test_name, test_dict = node
+      if _IsLeafNode(node):
+        if all(res not in test_dict['expected'].split() for res in
+               test_dict['actual'].split()):
+          failures.append(full_test_name)
+        else:
+          successes.append(full_test_name)
+      else:
+        for k in test_dict:
+          node_queues.append(
+            ('%s%s%s' % (full_test_name, delimiter, k),
+             test_dict[k]))
+    return successes, failures
+
+
+def load_tests(loader, tests, pattern):

[Ken Russell (switch to Gerrit), 2017/02/22 00:41:45]

Hmm. Does this test run correctly as part of gpu_unittests after the typ
cutover?

[nednguyen, 2017/02/22 00:46:06]

Yes it does. In the typ change, I have logic that does basically:

def LoadAllTestsInModule:
   if test is in invoked in integration harness:
      generate & return actual tests
   else:
      return []

[Ken Russell (switch to Gerrit), 2017/02/22 00:57:20]

It looks like something's wrong. See this dry run result:
https://build.chromium.org/p/tryserver.chromium.mac/builders/mac_chromium_rel...

specifically for telemetry_gpu_unittests:
https://build.chromium.org/p/tryserver.chromium.mac/builders/mac_chromium_rel...

Right now (before this CL), when running ./content/test/gpu/run_unittests.py ,
there are lines in the output like:
[0/33]
gpu_tests.gpu_integration_test_unittest.GpuIntegrationTestUnittest.testIntegrationUnittestWithBrowserCrashUponStart

But none of the tests from gpu_integration_test_unittest.py are running in that
test step any more.

+  del loader, tests, pattern  # Unused.
+  return gpu_integration_test.LoadAllTestsInModule(
+      sys.modules[__name__])