telemetry: Create a helper class to quickly extend profiles.

tools/perf/profile_creators/fast_navigation_profile_extender.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import time
+
+from telemetry.core import browser_finder
+from telemetry.core import browser_finder_exceptions
+from telemetry.core import exceptions
+from telemetry.core import util
+
+
+class FastNavigationProfileExtender(object):
+  """
+  This class creates or extends an existing profile by performing a set of tab
+  navigations in large batches. This is accomplished by opening a large number
+  of tabs, simultaneously navigating all the tabs, and then waiting for all the
+  tabs to load. This provides two benefits:
+    - Takes advantage of the high number of logical cores on modern CPUs.
+    - The total time spent waiting for navigations to time out scales linearly
+      with the number of batches, but does not scale with the size of the
+      batch.
+  """
+  def __init__(self):
+    super(FastNavigationProfileExtender, self).__init__()
+
+    # A reference to the browser that will be performing all of the tab
+    # navigations.
+    self._browser = None
+
+    # A static copy of the urls that this class is going to navigate to.
+    self._navigation_urls = None
+
+    # The number of tabs to use.
+    self._NUM_TABS = 15
+
+    # The number of pages to load in parallel.
+    self._NUM_PARALLEL_PAGES = 15
+
+    # It doesn't make sense for the batch size to be larger than the number of
+    # available tabs.
+    assert(self._NUM_PARALLEL_PAGES <= self._NUM_TABS)
+
+    # The amount of time to wait for pages to finish loading.
+    self._PAGE_LOAD_TIMEOUT_IN_SECONDS = 10
+
+    # The amount of time to wait for the retrieval of the URL of a tab.
+    self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS = 1
+
+    # The amount of time to wait for a navigation to be committed.
+    self._NAVIGATION_COMMIT_WAIT_IN_SECONDS = 0.1
+
+    # A list of tuples (tab, initial_url). A navigation command has been sent
+    # to |tab|. |tab| had a URL of |initial_url| before the command was sent.
+    self._queued_tabs = []

[nednguyen, 2015/02/10 18:48:43]

I don't think you need this either.

[erikchen, 2015/02/10 21:30:11]

Done.

+
+  def Run(self, finder_options):
+    """
+    |finder_options| contains the directory of the input profile, the directory
+    to place the output profile, and sufficient information to choose a specific
+    browser binary.
+    """
+    try:
+      self._navigation_urls = self.GetUrlsToNavigate()
+      self._SetUpBrowser(finder_options)
+      self._PerformNavigations()
+    finally:
+      self._TearDownBrowser()
+
+  def GetUrlsToNavigate(self):
+    """
+    Intended for subclass override. Returns a list of urls to be navigated to.
+    """
+    raise NotImplementedError()
+
+
+  def _GetPossibleBrowser(self, finder_options):
+    """Return a possible_browser with the given options."""
+    possible_browser = browser_finder.FindBrowser(finder_options)
+    if not possible_browser:
+      raise browser_finder_exceptions.BrowserFinderException(
+          'No browser found.\n\nAvailable browsers:\n%s\n' %
+          '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
+    finder_options.browser_options.browser_type = (
+        possible_browser.browser_type)
+
+    return possible_browser
+
+  def _RetrieveTabUrl(self, tab):
+    """Retrives the URL of the tab."""
+    try:
+      return tab.EvaluateJavaScript('document.URL',
+          self._TAB_URL_RETRIEVAL_TIMEOUT_IN_SECONDS)
+    except exceptions.DevtoolsTargetCrashException:
+      return None
+
+  def _BatchNavigateTabs(self, batch):
+    """
+    Performs a batch of tab navigations with minimal delay.
+
+    |batch| is a list of tuples (tab, url).
+    """
+    timeout_in_seconds = 0
+
+    for tab, url in batch:
+      initial_url = self._RetrieveTabUrl(tab)

[nednguyen, 2015/02/10 18:48:43]

Not sure that you need to do this kind of checking. Note that Navigate's
implementation does WaitForNavigate 

https://code.google.com/p/chromium/codesearch#chromium/src/tools/telemetry/te...

[erikchen, 2015/02/10 21:30:11]

This is necessary since I navigate the tab with 0 timeout, so the behavior is
not well defined. (It may navigate, it may start navigating, or most likely, it
may not even reflect that the navigation has started).

[nednguyen, 2015/02/12 02:10:53]

If the behavior of timeout_in_seconds=0 is not well defined, I think we should
improve the documentation in WebContent.Navigate(..) to reflect what happen when
timeout_in_seconds is 0, and make sure that it does what you want.

Maybe you can add an 'if' in InspectorPage.Navigate(...) to skip the wait when
timeout is 0.

+
+      try:
+        tab.Navigate(url, None, timeout_in_seconds)
+      except exceptions.DevtoolsTargetCrashException:
+        # We expect a time out, and don't mind if the webpage crashes. Ignore
+        # both exceptions.
+        pass
+
+      self._queued_tabs.append((tab, initial_url))
+
+  def _WaitForQueuedTabsToLoad(self):
+    """Waits for all the batch navigated tabs to finish loading."""
+    end_time = time.time() + self._PAGE_LOAD_TIMEOUT_IN_SECONDS
+    for tab, initial_url in self._queued_tabs:
+      seconds_to_wait = end_time - time.time()
+      seconds_to_wait = max(0, seconds_to_wait)
+
+      if seconds_to_wait == 0:
+        break
+
+      # Since we don't wait any time for the tab url navigation to commit, it's
+      # possible that the tab hasn't started navigating yet.
+      current_url = self._RetrieveTabUrl(tab)
+
+      if current_url == initial_url:
+        # If the navigation hasn't been committed yet, wait a small amount of
+        # time. Don't bother rechecking the condition, since it's also possible
+        # that the web page isn't processing javascript.
+        time.sleep(self._NAVIGATION_COMMIT_WAIT_IN_SECONDS)
+
+      try:
+        tab.WaitForDocumentReadyStateToBeComplete(seconds_to_wait)
+      except (util.TimeoutException, exceptions.DevtoolsTargetCrashException):
+        # Ignore time outs and web page crashes.
+        pass
+    self._queued_tabs = []
+
+  def _SetUpBrowser(self, finder_options):
+    """
+    Finds the browser, starts the browser, and opens the requisite number of
+    tabs.
+    """
+    possible_browser = self._GetPossibleBrowser(finder_options)
+    self._browser = possible_browser.Create(finder_options)
+
+    for _ in range(self._NUM_TABS):
+      self._browser.tabs.New()
+
+  def _PerformNavigations(self):
+    """
+    Performs the navigations specified by |_navigation_urls| in large batches.
+    """
+    # The index of the first url that has not yet been navigated to.
+    navigation_url_index = 0
+    while True:
+      # Generate the next batch of navigations.
+      batch = []
+      max_index = min(navigation_url_index + self._NUM_PARALLEL_PAGES,
+          len(self._navigation_urls))
+      for i in range(navigation_url_index, max_index):
+        url = self._navigation_urls[i]
+        tab = self._browser.tabs[i % self._NUM_TABS]
+        batch.append((tab, url))
+      navigation_url_index = max_index
+
+      self._BatchNavigateTabs(batch)
+      self._WaitForQueuedTabsToLoad()
+
+      if navigation_url_index == len(self._navigation_urls):
+        break
+
+  def _TearDownBrowser(self):
+    """
+    Teardown that is guaranteed to be executed before the instance is destroyed.
+    """
+    if self._browser:
+      self._browser.Close()
+      self._browser = None