Plumb Telemetry test name through to results object creation

tools/telemetry/telemetry/page/page_runner_unittest.py

-# Copyright 2012 The Chromium Authors. All rights reserved.
+# Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 import tempfile
 import unittest
 
+from telemetry import benchmark
 from telemetry import decorators
 from telemetry.core import browser_finder
 from telemetry.core import exceptions
 from telemetry.core import user_agent
 from telemetry.core import util
 from telemetry.page import page as page_module
 from telemetry.page import page_measurement
 from telemetry.page import page_set
 from telemetry.page import page_test
 from telemetry.page import page_runner
 from telemetry.page import test_expectations
+from telemetry.results import results_options
 from telemetry.unittest import options_for_unittests
 from telemetry.value import scalar
 from telemetry.value import string
 
 
 SIMPLE_CREDENTIALS_STRING = """
 {
   "test": {
     "username": "example",
     "password": "asdf"
   }
 }
 """
 
 
 def SetUpPageRunnerArguments(options):
   parser = options.CreateParser()
   page_runner.AddCommandLineArgs(parser)
   options.MergeDefaultValues(parser.get_default_values())
   page_runner.ProcessCommandLineArgs(parser, options)
 
+class EmptyMetadataForTest(benchmark.BenchmarkMetadata):
+  def __init__(self):
+    super(EmptyMetadataForTest, self).__init__('')
 
 class StubCredentialsBackend(object):
   def __init__(self, login_return_value):
     self.did_get_login = False
     self.did_get_login_no_longer_needed = False
     self.login_return_value = login_return_value
 
   @property
   def credentials_type(self):
     return 'test'
@@ skipping 20 matching lines @@
     page1 = page_module.Page('chrome://crash', ps)
     ps.pages.append(page1)
 
     class Test(page_test.PageTest):
       def ValidatePage(self, *args):
         pass
 
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(Test(), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(Test(), ps, expectations, options, results)
     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(1, len(results.failures))
 
   def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     ps.pages.append(page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
     ps.pages.append(page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
 
     class ExpectedException(Exception):
       pass
 
     class Test(page_test.PageTest):
       def __init__(self, *args):
         super(Test, self).__init__(*args)
         self.run_count = 0
       def ValidatePage(self, *_):
         old_run_count = self.run_count
         self.run_count += 1
         if old_run_count == 0:
           raise ExpectedException()
 
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     test = Test()
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
     self.assertEquals(2, test.run_count)
     self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(1, len(results.failures))
 
   def testHandlingOfCrashedTabWithExpectedFailure(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     expectations.Fail('chrome://crash')
     page1 = page_module.Page('chrome://crash', ps)
     ps.pages.append(page1)
 
     class Test(page_test.PageTest):
       def ValidatePage(self, *_):
         pass
 
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(
-        Test(), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(Test(), ps, expectations, options, results)
     self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(0, len(results.failures))
 
   def testRetryOnBrowserCrash(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     ps.pages.append(page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
 
     class CrashyMeasurement(page_measurement.PageMeasurement):
       has_crashed = False
       def MeasurePage(self, page, tab, results):
         # This value should be discarded on the first run when the
         # browser crashed.
         results.AddValue(
             string.StringValue(page, 'test', 't', str(self.has_crashed)))
         if not self.has_crashed:
           self.has_crashed = True
           raise exceptions.BrowserGoneException(tab.browser)
 
     options = options_for_unittests.GetCopy()
     options.output_format = 'csv'
 
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(CrashyMeasurement(), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(CrashyMeasurement(), ps, expectations, options, results)
 
     self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(0, len(results.failures))
     self.assertEquals(1, len(results.all_page_specific_values))
     self.assertEquals(
         'True', results.all_page_specific_values[0].GetRepresentativeString())
 
   @decorators.Disabled('xp')  # Flaky, http://crbug.com/390079.
   def testDiscardFirstResult(self):
     ps = page_set.PageSet()
@@ skipping 13 matching lines @@
 
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     options.reset_results = None
     options.upload_results = None
     options.results_label = None
 
     options.page_repeat = 1
     options.pageset_repeat = 1
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(Measurement(), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(Measurement(), ps, expectations, options, results)
     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(0, len(results.failures))
     self.assertEquals(0, len(results.all_page_specific_values))
 
     options.page_repeat = 1
     options.pageset_repeat = 2
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(Measurement(), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(Measurement(), ps, expectations, options, results)
     self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(0, len(results.failures))
     self.assertEquals(2, len(results.all_page_specific_values))
 
     options.page_repeat = 2
     options.pageset_repeat = 1
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(Measurement(), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(Measurement(), ps, expectations, options, results)
     self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(0, len(results.failures))
     self.assertEquals(2, len(results.all_page_specific_values))
 
     options.output_format = 'html'
     options.page_repeat = 1
     options.pageset_repeat = 1
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(Measurement(), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(Measurement(), ps, expectations, options, results)
     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(0, len(results.failures))
     self.assertEquals(0, len(results.all_page_specific_values))
 
   @decorators.Disabled('win')
   def testPagesetRepeat(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     ps.pages.append(page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
@@ skipping 12 matching lines @@
       options = options_for_unittests.GetCopy()
       options.output_format = 'buildbot'
       options.output_file = output_file
       options.reset_results = None
       options.upload_results = None
       options.results_label = None
 
       options.page_repeat = 1
       options.pageset_repeat = 2
       SetUpPageRunnerArguments(options)
-      results = page_runner.Run(Measurement(), ps, expectations, options)
+      results = results_options.CreateResults(EmptyMetadataForTest(), options)
+      page_runner.Run(Measurement(), ps, expectations, options, results)
       results.PrintSummary()
       self.assertEquals(4, len(GetSuccessfulPageRuns(results)))
       self.assertEquals(0, len(results.failures))
       with open(output_file) as f:
         stdout = f.read()
       self.assertIn('RESULT metric: blank.html= [1,3] unit', stdout)
       self.assertIn('RESULT metric: green_rect.html= [2,4] unit', stdout)
       self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout)
     finally:
       # TODO(chrishenry): This is a HACK!!1 Really, the right way to
@@ skipping 40 matching lines @@
         def DidStartBrowser(self, browser):
           browser.credentials.AddBackend(self._credentials_backend)
 
         def ValidatePage(self, *_):
           did_run[0] = True
 
       test = TestThatInstallsCredentialsBackend(credentials_backend)
       options = options_for_unittests.GetCopy()
       options.output_format = 'none'
       SetUpPageRunnerArguments(options)
-      page_runner.Run(test, ps, expectations, options)
+      results = results_options.CreateResults(EmptyMetadataForTest(), options)
+      page_runner.Run(test, ps, expectations, options, results)
     finally:
       os.remove(f.name)
 
     return did_run[0]
 
   def testUserAgent(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     page = page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
     ps.pages.append(page)
     ps.user_agent_type = 'tablet'
 
     class TestUserAgent(page_test.PageTest):
       def ValidatePage(self, _1, tab, _2):
         actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent')
         expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
         assert actual_user_agent.strip() == expected_user_agent
 
         # This is so we can check later that the test actually made it into this
         # function. Previously it was timing out before even getting here, which
         # should fail, but since it skipped all the asserts, it slipped by.
         self.hasRun = True # pylint: disable=W0201
 
     test = TestUserAgent()
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
 
     self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
 
   # Ensure that page_runner forces exactly 1 tab before running a page.
   @decorators.Enabled('has tabs')
   def testOneTab(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     page = page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
     ps.pages.append(page)
 
     class TestOneTab(page_test.PageTest):
       def __init__(self):
         super(TestOneTab, self).__init__()
         self._browser = None
 
       def DidStartBrowser(self, browser):
         self._browser = browser
         self._browser.tabs.New()
 
       def ValidatePage(self, *_):
         assert len(self._browser.tabs) == 1
 
     test = TestOneTab()
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
 
   # Ensure that page_runner allows the test to customize the browser before it
   # launches.
   def testBrowserBeforeLaunch(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     page = page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
     ps.pages.append(page)
 
@@ skipping 11 matching lines @@
         assert self._did_call_will_start
         self._did_call_did_start = True
 
       def ValidatePage(self, *_):
         assert self._did_call_did_start
 
     test = TestBeforeLaunch()
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
 
   def testRunPageWithStartupUrl(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     expectations = test_expectations.TestExpectations()
     page = page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
     page.startup_url = 'about:blank'
     ps.pages.append(page)
 
     class Measurement(page_measurement.PageMeasurement):
       def __init__(self):
         super(Measurement, self).__init__()
         self.browser_restarted = False
 
       def CustomizeBrowserOptionsForSinglePage(self, ps, options):
         self.browser_restarted = True
         super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps,
                                                                       options)
       def MeasurePage(self, page, tab, results):
         pass
 
     options = options_for_unittests.GetCopy()
     options.page_repeat = 2
     options.output_format = 'none'
     if not browser_finder.FindBrowser(options):
       return
     test = Measurement()
     SetUpPageRunnerArguments(options)
-    page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
     self.assertEquals('about:blank', options.browser_options.startup_url)
     self.assertTrue(test.browser_restarted)
 
   # Ensure that page_runner calls cleanUp when a page run fails.
   def testCleanUpPage(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
     page = page_module.Page(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
     ps.pages.append(page)
 
     class Test(page_test.PageTest):
       def __init__(self):
         super(Test, self).__init__()
         self.did_call_clean_up = False
 
       def ValidatePage(self, *_):
         raise exceptions.IntentionalException
 
       def CleanUpAfterPage(self, page, tab):
         self.did_call_clean_up = True
 
 
     test = Test()
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
     assert test.did_call_clean_up
 
   # Ensure skipping the test if page cannot be run on the browser
   def testPageCannotRunOnBrowser(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()
 
     class PageThatCannotRunOnBrowser(page_module.Page):
 
       def __init__(self):
@@ skipping 15 matching lines @@
       def ValidatePage(self, *args):
         pass
 
       def WillNavigateToPage(self, _1, _2):
         self.will_navigate_to_page_called = True
 
     test = Test()
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
     self.assertFalse(test.will_navigate_to_page_called)
     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
     self.assertEquals(0, len(results.failures))
 
   def TestUseLiveSitesFlag(self, options, expect_from_archive):
     ps = page_set.PageSet(
       file_path=util.GetUnittestDataDir(),
       archive_data_file='data/archive_blank.json')
     ps.pages.append(page_module.Page(
       'file://blank.html', ps, base_dir=ps.base_dir))
     expectations = test_expectations.TestExpectations()
 
     class ArchiveTest(page_measurement.PageMeasurement):
       def __init__(self):
         super(ArchiveTest, self).__init__()
         self.is_page_from_archive = False
         self.archive_path_exist = True
 
       def WillNavigateToPage(self, page, tab):
         self.archive_path_exist = (page.archive_path
                                    and os.path.isfile(page.archive_path))
         self.is_page_from_archive = (
           tab.browser._wpr_server is not None) # pylint: disable=W0212
 
       def MeasurePage(self, _, __, results):
         pass
 
     test = ArchiveTest()
-    page_runner.Run(test, ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(test, ps, expectations, options, results)
     if expect_from_archive and not test.archive_path_exist:
       logging.warning('archive path did not exist, asserting that page '
                       'is from archive is skipped.')
       return
     self.assertEquals(expect_from_archive, test.is_page_from_archive)
 
   def testUseLiveSitesFlagSet(self):
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     options.use_live_sites = True
@@ skipping 34 matching lines @@
     page4 = TestPage(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
     ps.pages.append(page4)
     page5 = TestPage(
         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
     ps.pages.append(page5)
 
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     SetUpPageRunnerArguments(options)
-    results = page_runner.Run(Test(max_failures=2), ps, expectations, options)
+    results = results_options.CreateResults(EmptyMetadataForTest(), options)
+    page_runner.Run(Test(max_failures=2), ps, expectations, options, results)
     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
     # Runs up to max_failures+1 failing tests before stopping, since
     # every tests after max_failures failures have been encountered
     # may all be passing.
     self.assertEquals(3, len(results.failures))
     self.assertTrue(page1.was_run)
     self.assertTrue(page2.was_run)
     self.assertTrue(page3.was_run)
     self.assertFalse(page4.was_run)
     self.assertFalse(page5.was_run)