[Telemetry] Introduce shared_user_story_state for real.

tools/telemetry/telemetry/page/page_runner.py

-#  Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import logging
-import optparse
-import os
-import random
-import sys
-import time
-
-from telemetry import decorators
-from telemetry.core import exceptions
-from telemetry.core import util
-from telemetry.core import wpr_modes
-from telemetry.page import shared_page_state
-from telemetry.page import page_test
-from telemetry.page.actions import page_action
-from telemetry.results import results_options
-from telemetry.user_story import user_story_filter
-from telemetry.util import cloud_storage
-from telemetry.util import exception_formatter
-from telemetry.value import failure
-from telemetry.value import skip
-
-
-def AddCommandLineArgs(parser):
-  user_story_filter.UserStoryFilter.AddCommandLineArgs(parser)
-  results_options.AddResultsOptions(parser)
-
-  # Page set options
-  group = optparse.OptionGroup(parser, 'Page set ordering and repeat options')
-  group.add_option('--pageset-shuffle', action='store_true',
-      dest='pageset_shuffle',
-      help='Shuffle the order of pages within a pageset.')
-  group.add_option('--pageset-shuffle-order-file',
-      dest='pageset_shuffle_order_file', default=None,
-      help='Filename of an output of a previously run test on the current '
-      'pageset. The tests will run in the same order again, overriding '
-      'what is specified by --page-repeat and --pageset-repeat.')
-  group.add_option('--page-repeat', default=1, type='int',
-                   help='Number of times to repeat each individual page '
-                   'before proceeding with the next page in the pageset.')
-  group.add_option('--pageset-repeat', default=1, type='int',
-                   help='Number of times to repeat the entire pageset.')
-  group.add_option('--max-failures', default=None, type='int',
-                   help='Maximum number of test failures before aborting '
-                   'the run. Defaults to the number specified by the '
-                   'PageTest.')
-  parser.add_option_group(group)
-
-  # WPR options
-  group = optparse.OptionGroup(parser, 'Web Page Replay options')
-  group.add_option('--use-live-sites',
-      dest='use_live_sites', action='store_true',
-      help='Run against live sites and ignore the Web Page Replay archives.')
-  parser.add_option_group(group)
-
-  parser.add_option('-d', '--also-run-disabled-tests',
-                    dest='run_disabled_tests',
-                    action='store_true', default=False,
-                    help='Ignore @Disabled and @Enabled restrictions.')
-
-def ProcessCommandLineArgs(parser, args):
-  user_story_filter.UserStoryFilter.ProcessCommandLineArgs(parser, args)
-  results_options.ProcessCommandLineArgs(parser, args)
-
-  # Page set options
-  if args.pageset_shuffle_order_file and not args.pageset_shuffle:
-    parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.')
-
-  if args.page_repeat < 1:
-    parser.error('--page-repeat must be a positive integer.')
-  if args.pageset_repeat < 1:
-    parser.error('--pageset-repeat must be a positive integer.')
-
-
-def _RunPageAndHandleExceptionIfNeeded(test, page_set, expectations,
-                                       page, results, state):
-  expectation = None
-  def ProcessError():
-    if expectation == 'fail':
-      msg = 'Expected exception while running %s' % page.url
-      exception_formatter.PrintFormattedException(msg=msg)
-    else:
-      msg = 'Exception while running %s' % page.url
-      results.AddValue(failure.FailureValue(page, sys.exc_info()))
-
-  try:
-    state.WillRunPage(page, page_set)
-    expectation, skip_value = state.GetPageExpectationAndSkipValue(expectations)
-    if expectation == 'skip':
-      assert skip_value
-      results.AddValue(skip_value)
-      return
-    state.RunPage(results)
-  except page_test.TestNotSupportedOnPlatformFailure:
-    raise
-  except (page_test.Failure, util.TimeoutException, exceptions.LoginException,
-          exceptions.ProfilingException):
-    ProcessError()
-  except exceptions.AppCrashException:
-    ProcessError()
-    state.TearDown(results)
-    if test.is_multi_tab_test:
-      logging.error('Aborting multi-tab test after browser or tab crashed at '
-                    'page %s' % page.url)
-      test.RequestExit()
-      return
-  except page_action.PageActionNotSupported as e:
-    results.AddValue(skip.SkipValue(page, 'Unsupported page action: %s' % e))
-  except Exception:
-    exception_formatter.PrintFormattedException(
-        msg='Unhandled exception while running %s' % page.url)
-    results.AddValue(failure.FailureValue(page, sys.exc_info()))
-  else:
-    if expectation == 'fail':
-      logging.warning('%s was expected to fail, but passed.\n', page.url)
-  finally:
-    state.DidRunPage(results)
-
-
-@decorators.Cache
-def _UpdatePageSetArchivesIfChanged(page_set):
-  # Scan every serving directory for .sha1 files
-  # and download them from Cloud Storage. Assume all data is public.
-  all_serving_dirs = page_set.serving_dirs.copy()
-  # Add individual page dirs to all serving dirs.
-  for page in page_set:
-    if page.is_file:
-      all_serving_dirs.add(page.serving_dir)
-  # Scan all serving dirs.
-  for serving_dir in all_serving_dirs:
-    if os.path.splitdrive(serving_dir)[1] == '/':
-      raise ValueError('Trying to serve root directory from HTTP server.')
-    for dirpath, _, filenames in os.walk(serving_dir):
-      for filename in filenames:
-        path, extension = os.path.splitext(
-            os.path.join(dirpath, filename))
-        if extension != '.sha1':
-          continue
-        cloud_storage.GetIfChanged(path, page_set.bucket)
-
-
-def Run(test, page_set, expectations, finder_options, results):
-  """Runs a given test against a given page_set with the given options."""
-  test.ValidatePageSet(page_set)
-
-  # Reorder page set based on options.
-  pages = _ShuffleAndFilterPageSet(page_set, finder_options)
-
-  if not finder_options.use_live_sites:
-    if finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD:
-      _UpdatePageSetArchivesIfChanged(page_set)
-      pages = _CheckArchives(page_set, pages, results)
-
-  for page in list(pages):
-    if not test.CanRunForPage(page):
-      results.WillRunPage(page)
-      logging.debug('Skipping test: it cannot run for %s', page.url)
-      results.AddValue(skip.SkipValue(page, 'Test cannot run'))
-      results.DidRunPage(page)
-      pages.remove(page)
-
-  if not pages:
-    return
-
-  state = shared_page_state.SharedPageState(test, finder_options, page_set)
-  pages_with_discarded_first_result = set()
-  max_failures = finder_options.max_failures  # command-line gets priority
-  if max_failures is None:
-    max_failures = test.max_failures  # may be None
-
-  try:
-    test.WillRunTest(finder_options)
-    for _ in xrange(finder_options.pageset_repeat):
-      for page in pages:
-        if test.IsExiting():
-          break
-        for _ in xrange(finder_options.page_repeat):
-          results.WillRunPage(page)
-          try:
-            _WaitForThermalThrottlingIfNeeded(state.platform)
-            _RunPageAndHandleExceptionIfNeeded(
-                test, page_set, expectations, page, results, state)
-          except Exception:
-            # Tear down & restart the state for unhandled exceptions thrown by
-            # _RunPageAndHandleExceptionIfNeeded.
-            results.AddValue(failure.FailureValue(page, sys.exc_info()))
-            state.TearDown(results)
-            state = shared_page_state.SharedPageState(
-                test, finder_options, page_set)
-          finally:
-            _CheckThermalThrottling(state.platform)
-            discard_run = (test.discard_first_result and
-                           page not in pages_with_discarded_first_result)
-            if discard_run:
-              pages_with_discarded_first_result.add(page)
-            results.DidRunPage(page, discard_run=discard_run)
-        if max_failures is not None and len(results.failures) > max_failures:
-          logging.error('Too many failures. Aborting.')
-          test.RequestExit()
-  finally:
-    state.TearDown(results)
-
-
-def _ShuffleAndFilterPageSet(page_set, finder_options):
-  if finder_options.pageset_shuffle_order_file:
-    return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file)
-  pages = [page for page in page_set.pages[:]
-           if user_story_filter.UserStoryFilter.IsSelected(page)]
-  if finder_options.pageset_shuffle:
-    random.shuffle(pages)
-  return pages
-
-
-def _CheckArchives(page_set, pages, results):
-  """Returns a subset of pages that are local or have WPR archives.
-
-  Logs warnings if any are missing.
-  """
-  # Warn of any problems with the entire page set.
-  if any(not p.is_local for p in pages):
-    if not page_set.archive_data_file:
-      logging.warning('The page set is missing an "archive_data_file" '
-                      'property. Skipping any live sites. To include them, '
-                      'pass the flag --use-live-sites.')
-    if not page_set.wpr_archive_info:
-      logging.warning('The archive info file is missing. '
-                      'To fix this, either add svn-internal to your '
-                      '.gclient using http://goto/read-src-internal, '
-                      'or create a new archive using record_wpr.')
-
-  # Warn of any problems with individual pages and return valid pages.
-  pages_missing_archive_path = []
-  pages_missing_archive_data = []
-  valid_pages = []
-  for page in pages:
-    if not page.is_local and not page.archive_path:
-      pages_missing_archive_path.append(page)
-    elif not page.is_local and not os.path.isfile(page.archive_path):
-      pages_missing_archive_data.append(page)
-    else:
-      valid_pages.append(page)
-  if pages_missing_archive_path:
-    logging.warning('The page set archives for some pages do not exist. '
-                    'Skipping those pages. To fix this, record those pages '
-                    'using record_wpr. To ignore this warning and run '
-                    'against live sites, pass the flag --use-live-sites.')
-  if pages_missing_archive_data:
-    logging.warning('The page set archives for some pages are missing. '
-                    'Someone forgot to check them in, or they were deleted. '
-                    'Skipping those pages. To fix this, record those pages '
-                    'using record_wpr. To ignore this warning and run '
-                    'against live sites, pass the flag --use-live-sites.')
-  for page in pages_missing_archive_path + pages_missing_archive_data:
-    results.WillRunPage(page)
-    results.AddValue(failure.FailureValue.FromMessage(
-        page, 'Page set archive doesn\'t exist.'))
-    results.DidRunPage(page)
-  return valid_pages
-
-
-def _WaitForThermalThrottlingIfNeeded(platform):
-  if not platform.CanMonitorThermalThrottling():
-    return
-  thermal_throttling_retry = 0
-  while (platform.IsThermallyThrottled() and
-         thermal_throttling_retry < 3):
-    logging.warning('Thermally throttled, waiting (%d)...',
-                    thermal_throttling_retry)
-    thermal_throttling_retry += 1
-    time.sleep(thermal_throttling_retry * 2)
-
-  if thermal_throttling_retry and platform.IsThermallyThrottled():
-    logging.warning('Device is thermally throttled before running '
-                    'performance tests, results will vary.')
-
-
-def _CheckThermalThrottling(platform):
-  if not platform.CanMonitorThermalThrottling():
-    return
-  if platform.HasBeenThermallyThrottled():
-    logging.warning('Device has been thermally throttled during '
-                    'performance tests, results will vary.')