Move PageSet._InitializeArchive to PageRunner

tools/telemetry/telemetry/page/page_runner.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import copy
 import glob
 import logging
 import optparse
 import os
 import random
 import sys
 import tempfile
 import time
 
 from telemetry import decorators
 from telemetry.core import browser_finder
 from telemetry.core import exceptions
 from telemetry.core import util
 from telemetry.core import wpr_modes
 from telemetry.core.platform.profiler import profiler_finder
+from telemetry.page import cloud_storage
 from telemetry.page import page_filter
 from telemetry.page import page_runner_repeat
 from telemetry.page import page_test
 from telemetry.page import results_options
 from telemetry.page.actions import navigate
 from telemetry.page.actions import page_action
 from telemetry.util import exception_formatter
 
 
 class _RunState(object):
@@ skipping 271 matching lines @@
       state.StopBrowser()
       if not tries:
         logging.error('Lost connection to browser 3 times. Failing.')
         raise
       if test.is_multi_tab_test:
         logging.error(
           'Lost connection to browser during multi-tab test. Failing.')
         raise
 
 
+def _UpdatePageSetArchivesIfChanged(page_set):
+  # Attempt to download the credentials file.
+  if page_set.credentials_path:
+    try:
+      cloud_storage.GetIfChanged(
+          os.path.join(page_set.base_dir, page_set.credentials_path))
+    except (cloud_storage.CredentialsError, cloud_storage.PermissionError):
+      logging.warning('Cannot retrieve credential file: %s',
+                      page_set.credentials_path)
+  # Scan every serving directory for .sha1 files
+  # and download them from Cloud Storage. Assume all data is public.
+  all_serving_dirs = page_set.serving_dirs.copy()
+  # Add individual page dirs to all serving dirs.
+  for page in page_set:
+    if page.is_file:
+      all_serving_dirs.add(page.serving_dir)
+  # Scan all serving dirs.
+  for serving_dir in all_serving_dirs:
+    if os.path.splitdrive(serving_dir)[1] == '/':
+      raise ValueError('Trying to serve root directory from HTTP server.')
+    for dirpath, _, filenames in os.walk(serving_dir):
+      for filename in filenames:
+        path, extension = os.path.splitext(
+            os.path.join(dirpath, filename))
+        if extension != '.sha1':
+          continue
+        cloud_storage.GetIfChanged(path)
+
+
 def Run(test, page_set, expectations, finder_options):
   """Runs a given test against a given page_set with the given options."""
   results = results_options.PrepareResults(test, finder_options)
   browser_options = finder_options.browser_options
 
   test.ValidatePageSet(page_set)
 
   # Create a possible_browser with the given options.
   test.CustomizeBrowserOptions(finder_options)
   try:
@@ skipping 12 matching lines @@
 
   if not decorators.IsEnabled(
       test, browser_options.browser_type, possible_browser.platform):
     return results
 
   # Reorder page set based on options.
   pages = _ShuffleAndFilterPageSet(page_set, finder_options)
 
   if (not finder_options.use_live_sites and
       browser_options.wpr_mode != wpr_modes.WPR_RECORD):
+    _UpdatePageSetArchivesIfChanged(page_set)
     pages = _CheckArchives(page_set, pages, results)
 
   # Verify credentials path.
   credentials_path = None
   if page_set.credentials_path:
     credentials_path = os.path.join(os.path.dirname(page_set.file_path),
                                     page_set.credentials_path)
     if not os.path.exists(credentials_path):
       credentials_path = None
 
@@ skipping 208 matching lines @@
     logging.error('Device is thermally throttled before running '
                   'performance tests, results will vary.')
 
 
 def _CheckThermalThrottling(platform):
   if not platform.CanMonitorThermalThrottling():
     return
   if platform.HasBeenThermallyThrottled():
     logging.error('Device has been thermally throttled during '
                   'performance tests, results will vary.')