Have repeats understand "time"

tools/telemetry/telemetry/page/page_runner.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 import codecs
 import glob
 import logging
 import os
 import sys
 import time
 import traceback
@@ skipping 53 matching lines @@
 
       # Ensure only one tab is open.
       while len(self.browser.tabs) > 1:
         self.browser.tabs[-1].Close()
 
     if not self.tab:
       self.tab = self.browser.tabs[0]
 
     if self.first_page:
       self.first_page = False
-      test.WillRunPageSet(self.tab)
 
   def StopBrowser(self):
     self._is_tracing = False
 
     if self.tab:
       self.tab.Disconnect()
       self.tab = None
 
     if self.browser:
       self.browser.Close()
       self.browser = None
 
       # Restarting the state will also restart the wpr server. If we're
       # recording, we need to continue adding into the same wpr archive,
       # not overwrite it.
       self._append_to_existing_wpr = True
 
   def StartProfiling(self, page, options):
     output_file = os.path.join(options.profiler_dir, page.url_as_file_safe_name)
-    if options.page_repeat != 1 or options.pageset_repeat != 1:
+    if (options.page_repeat_iters != 1 or options.pageset_repeat_iters != 1 or
+        options.page_repeat_secs or options.pageset_repeat_secs):
       output_file = _GetSequentialFileName(output_file)
     self.browser.StartProfiling(options, output_file)
 
   def StopProfiling(self):
     self.browser.StopProfiling()
 
   def StartTracing(self):
     if not self.browser.supports_tracing:
       return
 
     self._is_tracing = True
     self.browser.StartTracing()
 
   def StopTracing(self, page, options):
     if not self._is_tracing:
       return
 
     assert self.browser
     self._is_tracing = False
     self.browser.StopTracing()
     trace_result = self.browser.GetTraceResultAndReset()
     logging.info('Processing trace...')
 
     trace_file = os.path.join(options.trace_dir, page.url_as_file_safe_name)
-    if options.page_repeat != 1 or options.pageset_repeat != 1:
+    if (options.page_repeat_iters != 1 or options.pageset_repeat_iters != 1 or
+        options.page_repeat_secs or options.pageset_repeat_secs):
       trace_file = _GetSequentialFileName(trace_file)
     trace_file += '.json'
 
     with codecs.open(trace_file, 'w',
                      encoding='utf-8') as trace_file:
       trace_result.Serialize(trace_file)
     logging.info('Trace saved.')
 
 
 class PageState(object):
@@ skipping 28 matching lines @@
 
   def CleanUpPage(self, page, tab):
     if page.credentials and self._did_login:
       tab.browser.credentials.LoginNoLongerNeeded(tab, page.credentials)
 
 
 def AddCommandLineOptions(parser):
   page_filter_module.PageFilter.AddCommandLineOptions(parser)
 
 
+def _PrepareAndRunPage(test, page_set, options, page, credentials_path,
+                       possible_browser, results, state):
+  if options.wpr_mode != wpr_modes.WPR_RECORD:
+    if page.archive_path and os.path.isfile(page.archive_path):
+      possible_browser.options.wpr_mode = wpr_modes.WPR_REPLAY
+    else:
+      possible_browser.options.wpr_mode = wpr_modes.WPR_OFF
+  results_for_current_run = results
+  if state.first_page and test.discard_first_result:
+    # If discarding results, substitute a dummy object.
+    results_for_current_run = type(results)()
+  results_for_current_run.StartTest(page)
+  tries = 3
+  while tries:
+    try:
+      state.StartBrowser(test, page_set, page, possible_browser,
+                         credentials_path, page.archive_path)
+
+      _WaitForThermalThrottlingIfNeeded(state.browser.platform)
+
+      if options.trace_dir:
+        state.StartTracing()
+      if options.profiler_dir:
+        state.StartProfiling(page, options)
+
+      try:
+        _RunPage(test, page, state.tab, results_for_current_run, options)
+        _CheckThermalThrottling(state.browser.platform)
+      except exceptions.TabCrashException:
+        stdout = ''
+        if not options.show_stdout:
+          stdout = state.browser.GetStandardOutput()
+          stdout = (('\nStandard Output:\n') +
+                    ('*' * 80) +
+                    '\n\t' + stdout.replace('\n', '\n\t') + '\n' +
+                    ('*' * 80))
+        logging.warning('Tab crashed: %s%s', page.url, stdout)
+        state.StopBrowser()
+
+      if options.trace_dir:
+        state.StopTracing(page, options)
+      if options.profiler_dir:
+        state.StopProfiling()
+
+      if test.NeedsBrowserRestartAfterEachRun(state.tab):
+        state.StopBrowser()
+
+      break
+    except exceptions.BrowserGoneException:
+      logging.warning('Lost connection to browser. Retrying.')
+      state.StopBrowser()
+      tries -= 1
+      if not tries:
+        logging.error('Lost connection to browser 3 times. Failing.')
+        raise
+  results_for_current_run.StopTest(page)
+
+
+def _RunRepeatLoop(test, page_set, options, credentials_path, possible_browser,
+                   results, state, pages):
+  """Repeats each page or pageset for a specified time or iteration limit.
+
+  Uses the '--pageset-repeat' and '--page-repeat' cmd line options.
+  """
+  pageset_start_time = time.time()
+  pageset_iters = 0
+  while True:
+    if (options.pageset_repeat_secs and
+        time.time() - pageset_start_time > options.pageset_repeat_secs):
+      break
+    elif (options.pageset_repeat_iters != 1 and
+          pageset_iters >= options.pageset_repeat_iters):
+      break
+    pageset_iters += 1
+    for page in pages:
+      page_start_time = time.time()
+      page_iters = 0
+      while True:
+        if (options.page_repeat_secs and
+            time.time() - page_start_time > options.page_repeat_secs):
+          break
+        elif (options.page_repeat_iters != 1 and
+              page_iters >= options.page_repeat_iters):
+          break
+        page_iters += 1
+        _PrepareAndRunPage(test, page_set, options, page, credentials_path,
+                           possible_browser, results, state)
+
+
 def Run(test, page_set, options):
   """Runs a given test against a given page_set with the given options."""
   results = test.PrepareResults(options)
 
   # Create a possible_browser with the given options.
   test.CustomizeBrowserOptions(options)
   possible_browser = browser_finder.FindBrowser(options)
   if not possible_browser:
     raise Exception('No browser found.\n'
         'Use --browser=list to figure out which are available.')
@@ skipping 18 matching lines @@
     options.browser_user_agent_type = page_set.user_agent_type
 
   for page in pages:
     test.CustomizeBrowserOptionsForPage(page, possible_browser.options)
 
   _ValidateOrCreateEmptyDirectory('--trace-dir', options.trace_dir)
   _ValidateOrCreateEmptyDirectory('--profiler-dir', options.profiler_dir)
 
   state = _RunState()
   # TODO(dtu): Move results creation and results_for_current_run into RunState.
-  results_for_current_run = results
 
   try:
-    for page in pages:
-      if options.wpr_mode != wpr_modes.WPR_RECORD:
-        if page.archive_path and os.path.isfile(page.archive_path):
-          possible_browser.options.wpr_mode = wpr_modes.WPR_REPLAY
-        else:
-          possible_browser.options.wpr_mode = wpr_modes.WPR_OFF
-      results_for_current_run = results
-      if state.first_page and test.discard_first_result:
-        # If discarding results, substitute a dummy object.
-        results_for_current_run = type(results)()
-      results_for_current_run.StartTest(page)
-      tries = 3
-      while tries:
-        try:
-          state.StartBrowser(test, page_set, page, possible_browser,
-                             credentials_path, page.archive_path)
-
-          _WaitForThermalThrottlingIfNeeded(state.browser.platform)
-
-          if options.trace_dir:
-            state.StartTracing()
-          if options.profiler_dir:
-            state.StartProfiling(page, options)
-
-          try:
-            _RunPage(test, page, state.tab, results_for_current_run, options)
-            _CheckThermalThrottling(state.browser.platform)
-          except exceptions.TabCrashException:
-            stdout = ''
-            if not options.show_stdout:
-              stdout = state.browser.GetStandardOutput()
-              stdout = (('\nStandard Output:\n') +
-                        ('*' * 80) +
-                        '\n\t' + stdout.replace('\n', '\n\t') + '\n' +
-                        ('*' * 80))
-            logging.warning('Tab crashed: %s%s', page.url, stdout)
-            state.StopBrowser()
-
-          if options.trace_dir:
-            state.StopTracing(page, options)
-          if options.profiler_dir:
-            state.StopProfiling()
-
-          if test.NeedsBrowserRestartAfterEachRun(state.tab):
-            state.StopBrowser()
-
-          break
-        except exceptions.BrowserGoneException:
-          logging.warning('Lost connection to browser. Retrying.')
-          state.StopBrowser()
-          tries -= 1
-          if not tries:
-            logging.error('Lost connection to browser 3 times. Failing.')
-            raise
-      results_for_current_run.StopTest(page)
-    test.DidRunPageSet(state.tab, results_for_current_run)
+    test.WillRunTest(state.tab)
+    _RunRepeatLoop(test, page_set, options, credentials_path, possible_browser,
+                   results, state, pages)
+    test.DidRunTest(state.tab, results)
   finally:
     state.StopBrowser()
 
   return results
 
 
 def _ShuffleAndFilterPageSet(page_set, options):
   if options.pageset_shuffle_order_file and not options.pageset_shuffle:
     raise Exception('--pageset-shuffle-order-file requires --pageset-shuffle.')
 
   if options.pageset_shuffle_order_file:
     return page_set.ReorderPageSet(options.pageset_shuffle_order_file)
 
   page_filter = page_filter_module.PageFilter(options)
   pages = [page for page in page_set.pages[:]
            if not page.disabled and page_filter.IsSelected(page)]
 
   if options.pageset_shuffle:
     random.Random().shuffle(pages)
-  return [page
-      for _ in xrange(int(options.pageset_repeat))
-      for page in pages
-      for _ in xrange(int(options.page_repeat))]
+
+  return pages
 
 
 def _CheckArchives(page_set, pages, results):
   """Returns a subset of pages that are local or have WPR archives.
 
   Logs warnings if any are missing."""
   page_set_has_live_sites = False
   for page in pages:
     parsed_url = urlparse.urlparse(page.url)
     if parsed_url.scheme != 'chrome' and parsed_url.scheme != 'file':
@@ skipping 41 matching lines @@
   for page in pages_missing_archive_path + pages_missing_archive_data:
     results.StartTest(page)
     results.AddErrorMessage(page, 'Page set archive doesn\'t exist.')
     results.StopTest(page)
 
   return [page for page in pages if page not in
           pages_missing_archive_path + pages_missing_archive_data]
 
 
 def _RunPage(test, page, tab, results, options):
   if not test.CanRunForPage(page):

[edmundyan1, 2013/07/16 21:17:37]

Kind of a bug.  If we ran --page-repeat=100, we check if the page has that test
defined 100 times whereas one time is sufficient.  This is exasperated when we
run just --page-repeat=2s, where each "run" is milliseconds and we end up
skipping the page thousands of times.

     logging.warning('Skipping test: it cannot run for %s', page.url)
     results.AddSkip(page, 'Test cannot run')
     return
 
   logging.info('Running %s' % page.url)
 
   page_state = PageState()
 
   try:
     page_state.PreparePage(page, tab, test)
@@ skipping 56 matching lines @@
     logging.error('Device is thermally throttled before running '
                   'performance tests, results will vary.')
 
 
 def _CheckThermalThrottling(platform):
   if not platform.CanMonitorThermalThrottling():
     return
   if platform.HasBeenThermallyThrottled():
     logging.error('Device has been thermally throttled during '
                   'performance tests, results will vary.')