telemetry: Create a helper class to quickly extend profiles.

tools/perf/profile_creators/fast_navigation_profile_extender.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import time
+
+from telemetry.core import browser_finder
+from telemetry.core import browser_finder_exceptions
+from telemetry.core import exceptions
+from telemetry.core import util
+
+
+class FastNavigationProfileExtender(object):
+  """
+  This class creates or extends an existing profile by performing a set of tab
+  navigations in large batches. This is accomplished by opening a large number
+  of tabs, simultaneously navigating all the tabs, and then waiting for all the
+  tabs to load. This provides two benefits:
+    - Takes advantage of the high number of logical cores on modern CPUs.
+    - The total time spent waiting for navigations to time out scales linearly
+      with the number of batches, but does not scale with the size of the
+      batch.
+  """
+  def Run(self, finder_options):
+    """
+    Superclass override.
+
+    |finder_options| contains the directory of the input profile, the directory
+    to place the output profile, and sufficient information to choose a specific
+    browser binary.
+    """
+    profile_extender = _FastNavigationUserStory(finder_options,
+        self.NavigationUrls())
+    try:
+      profile_extender.WillRunUserStory()
+      profile_extender.RunUserStory()

[nednguyen, 2015/02/09 20:57:18]

You will only need these two hooks if you think they are extension points for
other profiler to implement the behavior differently.

[erikchen, 2015/02/09 21:17:35]

Renamed the hooks to BrowserSetup() and PerformNavigations()

+    except:
+      raise
+    finally:
+      profile_extender.TearDownState()

[nednguyen, 2015/02/09 20:57:18]

Also this one.

[erikchen, 2015/02/09 21:17:35]

Renamed to BrowserTeardown()

+
+  def NavigationUrls(self):
+    """
+    Intended for subclass override. Returns a list of urls to be navigated to.
+    """
+    raise NotImplementedError()
+
+
+class _FastNavigationUserStory(object):
+  """
+  This class contains the bulk of the logic related to
+  FastNavigationProfileExtender. Once UserStory has been refactored, this class
+  should become a subclass of UserStory. For more details, see
+  http://code.google.com/p/chromium/issues/detail?id=417812
+
+  This class intentionally mimics the format of SharedUserStoryState to make
+  the future refactor easier.
+  """
+  def __init__(self, finder_options, navigation_urls):
+    super(_FastNavigationUserStory, self).__init__()
+    self.browser = None
+    self._finder_options = finder_options
+    self._navigation_urls = navigation_urls
+
+    # The number of tabs to use.
+    self._NUM_TABS = 15
+
+    # The number of pages to load in parallel.
+    self._NUM_PARALLEL_PAGES = 15
+
+    # The amount of time to wait for pages to finish loading.
+    self._PAGE_LOAD_TIMEOUT_IN_SECONDS = 10
+
+    # The amount of time to wait for the retrieval of the URL of a tab.
+    self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS = 1
+
+    # The amount of time to wait for a navigation to be committed.
+    self._NAVIGATION_COMMIT_WAIT_IN_SECONDS = 0.1
+
+    # A list of tuples (tab, initial_url). A navigation command has been sent
+    # to |tab|. |tab| had a URL of |initial_url| before the command was sent.
+    self._queued_tabs = []
+
+    # The index of the first url that has not yet been navigated to.
+    self._navigation_url_index = 0
+
+  def _GetPossibleBrowser(self, finder_options):
+    """Return a possible_browser with the given options."""
+    possible_browser = browser_finder.FindBrowser(finder_options)
+    if not possible_browser:
+      raise browser_finder_exceptions.BrowserFinderException(
+          'No browser found.\n\nAvailable browsers:\n%s\n' %
+          '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
+    finder_options.browser_options.browser_type = (
+        possible_browser.browser_type)
+
+    return possible_browser
+
+  def _RetrieveTabUrl(self, tab):
+    """Retrives the URL of the tab."""
+    try:
+      return tab.EvaluateJavaScript('document.URL',
+          self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS)
+    except exceptions.DevtoolsTargetCrashException:
+      return None
+
+  def _BatchNavigateTabs(self):
+    """Performs a batch of tab navigations with minimal delay."""
+    max_index = min(self._navigation_url_index + self._NUM_PARALLEL_PAGES,
+        len(self._navigation_urls))
+    timeout_in_seconds = 0
+
+    for i in range(self._navigation_url_index, max_index):
+      url = self._navigation_urls[i]
+      tab = self.browser.tabs[i % self._NUM_TABS]
+      initial_url = self._RetrieveTabUrl(tab)
+
+      try:
+        tab.Navigate(url, None, timeout_in_seconds)
+      except exceptions.DevtoolsTargetCrashException:
+        # We expect a time out, and don't mind if the webpage crashes. Ignore
+        # both exceptions.
+        pass
+
+      self._queued_tabs.append((tab, initial_url))
+    self._navigation_url_index = max_index
+
+  def _WaitForQueuedTabsToLoad(self):
+    """Waits for all the batch navigated tabs to finish loading."""
+    end_time = time.time() + self._PAGE_LOAD_TIMEOUT_IN_SECONDS
+    for tab, initial_url in self._queued_tabs:
+      seconds_to_wait = end_time - time.time()
+      seconds_to_wait = max(0, seconds_to_wait)
+
+      if seconds_to_wait == 0:
+        break
+
+      # Since we don't wait any time for the tab url navigation to commit, it's
+      # possible that the tab hasn't started navigating yet.
+      current_url = self._RetrieveTabUrl(tab)
+
+      if current_url == initial_url:
+        # If the navigation hasn't been committed yet, wait a small amount of
+        # time. Don't bother rechecking the condition, since it's also possible
+        # that the web page isn't processing javascript.
+        time.sleep(self._NAVIGATION_COMMIT_WAIT_IN_SECONDS)
+
+      try:
+        tab.WaitForDocumentReadyStateToBeComplete(seconds_to_wait)
+      except (util.TimeoutException, exceptions.DevtoolsTargetCrashException):
+        # Ignore time outs and web page crashes.
+        pass
+    self._queued_tabs = []
+
+  def WillRunUserStory(self):
+    """
+    Finds the browser, starts the browser, and opens the requisite number of
+    tabs.
+    """
+    possible_browser = self._GetPossibleBrowser(self._finder_options)
+    self.browser = possible_browser.Create(self._finder_options)
+
+    for _ in range(self._NUM_TABS):
+      self.browser.tabs.New()
+
+  def RunUserStory(self):
+    """
+    Performs the navigations specified by |_navigation_urls| in large batches.
+    """
+    while True:
+      self._BatchNavigateTabs()
+      self._WaitForQueuedTabsToLoad()
+
+      if self._navigation_url_index == len(self._navigation_urls):
+        break
+
+  def TearDownState(self):
+    """
+    Teardown that is guaranteed to be executed before the instance is destroyed.
+    """
+    if self.browser:
+      self.browser.Close()
+      self.browser = None