Roll out framework change & update gpu tests to conform to the API changes

content/test/gpu/gpu_tests/gpu_integration_test_unittest.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import logging
-import mock
 import os
+import sys
 import tempfile
 import unittest
 
 from telemetry.testing import fakes
 from telemetry.testing import browser_test_runner
 
 import gpu_project_config
 
 from gpu_tests import gpu_integration_test
 from gpu_tests import gpu_test_expectations
 
 class SimpleIntegrationUnittest(gpu_integration_test.GpuIntegrationTest):
   # Must be class-scoped since instances aren't reused across runs.
   _num_flaky_runs_to_fail = 2
 
   _num_browser_starts = 0
 
   @classmethod
   def Name(cls):
     return 'simple_integration_unittest'
 
   def setUp(self):
     super(SimpleIntegrationUnittest, self).setUp()
 
   @classmethod
-  def setUpClass(cls):
+  def SetUpProcess(cls):
     finder_options = fakes.CreateBrowserFinderOptions()
     finder_options.browser_options.platform = fakes.FakeLinuxPlatform()
     finder_options.output_formats = ['none']
     finder_options.suppress_gtest_report = True
     finder_options.output_dir = None
     finder_options.upload_bucket = 'public'
     finder_options.upload_results = False
     cls._finder_options = finder_options
     cls.platform = None
     cls.browser = None
@@ skipping 33 matching lines @@
       raise Exception('Expected exception')
 
 
 class BrowserStartFailureIntegrationUnittest(
   gpu_integration_test.GpuIntegrationTest):
 
   _num_browser_crashes = 0
   _num_browser_starts = 0
 
   @classmethod
-  def setUpClass(cls):
+  def SetUpProcess(cls):
     cls._fake_browser_options = \
         fakes.CreateBrowserFinderOptions(execute_on_startup=cls.CrashOnStart)
     cls._fake_browser_options.browser_options.platform = \
         fakes.FakeLinuxPlatform()
     cls._fake_browser_options.output_formats = ['none']
     cls._fake_browser_options.suppress_gtest_report = True
     cls._fake_browser_options.output_dir = None
     cls._fake_browser_options .upload_bucket = 'public'
     cls._fake_browser_options .upload_results = False
     cls._finder_options = cls._fake_browser_options
@@ skipping 29 matching lines @@
     pass
 
 
 class BrowserCrashAfterStartIntegrationUnittest(
   gpu_integration_test.GpuIntegrationTest):
 
   _num_browser_crashes = 0
   _num_browser_starts = 0
 
   @classmethod
-  def setUpClass(cls):
+  def SetUpProcess(cls):
     cls._fake_browser_options = fakes.CreateBrowserFinderOptions(
       execute_after_browser_creation=cls.CrashAfterStart)
     cls._fake_browser_options.browser_options.platform = \
         fakes.FakeLinuxPlatform()
     cls._fake_browser_options.output_formats = ['none']
     cls._fake_browser_options.suppress_gtest_report = True
     cls._fake_browser_options.output_dir = None
     cls._fake_browser_options .upload_bucket = 'public'
     cls._fake_browser_options .upload_results = False
     cls._finder_options = cls._fake_browser_options
@@ skipping 29 matching lines @@
     yield ('restart', 'restart.html', ())
 
   def RunActualGpuTest(self, file_path, *args):
     # The logic of this test is run when the browser starts, it fails twice
     # and then succeeds on the third time so we are just testing that this
     # is successful based on the parameters.
     pass
 
 
 class GpuIntegrationTestUnittest(unittest.TestCase):
-  @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
-  def testSimpleIntegrationUnittest(self, mockInitDependencyManager):
+  def testSimpleIntegrationUnittest(self):
     self._RunIntegrationTest(
       'simple_integration_unittest', [
         'unexpected_error',
         'unexpected_failure'
       ], [
         'expected_failure',
         'expected_flaky',
       ])
     # It might be nice to be more precise about the order of operations
     # with these browser restarts, but this is at least a start.
     self.assertEquals(SimpleIntegrationUnittest._num_browser_starts, 6)
 
-  @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
-  def testIntegrationUnittestWithBrowserFailure(
-      self, mockInitDependencyManager):
+  def testIntegrationUnittestWithBrowserFailure(self):
     self._RunIntegrationTest(
       'browser_start_failure_integration_unittest', [], ['restart'])
     self.assertEquals( \
       BrowserStartFailureIntegrationUnittest._num_browser_crashes, 2)
     self.assertEquals( \
       BrowserStartFailureIntegrationUnittest._num_browser_starts, 3)
 
-  @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
-  def testIntegrationUnittestWithBrowserCrashUponStart(
-      self, mockInitDependencyManager):
+  def testIntegrationUnittestWithBrowserCrashUponStart(self):
     self._RunIntegrationTest(
       'browser_crash_after_start_integration_unittest', [], ['restart'])
     self.assertEquals( \
       BrowserCrashAfterStartIntegrationUnittest._num_browser_crashes, 2)
     self.assertEquals( \
       BrowserCrashAfterStartIntegrationUnittest._num_browser_starts, 3)
 
   def _RunIntegrationTest(self, test_name, failures, successes):
-    options = browser_test_runner.TestRunOptions()
-    # Suppress printing out information for passing tests.
-    options.verbosity = 0
     config = gpu_project_config.CONFIG
     temp_file = tempfile.NamedTemporaryFile(delete=False)
     temp_file.close()
     temp_file_name = temp_file.name
     try:
       browser_test_runner.Run(
-          config, options,
+          config,
           [test_name,
-           '--write-abbreviated-json-results-to=%s' % temp_file_name])
+           '--write-full-results-to=%s' % temp_file_name])
       with open(temp_file_name) as f:
         test_result = json.load(f)
-      self.assertEquals(test_result['failures'], failures)
-      self.assertEquals(test_result['successes'], successes)
-      self.assertEquals(test_result['valid'], True)
+      actual_successes, actual_failures = self._ExtracTestResults(test_result)
+      self.assertEquals(actual_failures, failures)
+      self.assertEquals(actual_successes, successes)
 
     finally:
       os.remove(temp_file_name)
+
+  def _ExtracTestResults(self, test_result):
+    delimiter = test_result['path_delimiter']
+    failures = []
+    successes = []
+    def _IsLeafNode(node):
+      test_dict = node[1]
+      return ('expected' in test_dict and
+              isinstance(test_dict['expected'], basestring))
+    node_queues = []
+    for t in test_result['tests']:
+      node_queues.append((t, test_result['tests'][t]))
+    while node_queues:
+      node = node_queues.pop()
+      full_test_name, test_dict = node
+      if _IsLeafNode(node):
+        if all(res not in test_dict['expected'].split() for res in
+               test_dict['actual'].split()):
+          failures.append(full_test_name)
+        else:
+          successes.append(full_test_name)
+      else:
+        for k in test_dict:
+          node_queues.append(
+            ('%s%s%s' % (full_test_name, delimiter, k),
+             test_dict[k]))
+    return successes, failures
+
+
+def load_tests(loader, tests, pattern):

[Ken Russell (switch to Gerrit), 2017/02/22 00:41:45]

Hmm. Does this test run correctly as part of gpu_unittests after the typ
cutover?

[nednguyen, 2017/02/22 00:46:06]

Yes it does. In the typ change, I have logic that does basically:

def LoadAllTestsInModule:
   if test is in invoked in integration harness:
      generate & return actual tests
   else:
      return []

[Ken Russell (switch to Gerrit), 2017/02/22 00:57:20]

It looks like something's wrong. See this dry run result:
https://build.chromium.org/p/tryserver.chromium.mac/builders/mac_chromium_rel...

specifically for telemetry_gpu_unittests:
https://build.chromium.org/p/tryserver.chromium.mac/builders/mac_chromium_rel...

Right now (before this CL), when running ./content/test/gpu/run_unittests.py ,
there are lines in the output like:
[0/33]
gpu_tests.gpu_integration_test_unittest.GpuIntegrationTestUnittest.testIntegrationUnittestWithBrowserCrashUponStart

But none of the tests from gpu_integration_test_unittest.py are running in that
test step any more.

+  del loader, tests, pattern  # Unused.
+  return gpu_integration_test.LoadAllTestsInModule(
+      sys.modules[__name__])