Add cache_temperature state "HOT"

telemetry/telemetry/page/cache_temperature.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Cache temperature specifies how the browser cache should be configured before
 the page run.
 
 See design doc for details:
 https://docs.google.com/document/u/1/d/12D7tkhZi887g9d0U2askU9JypU_wYiEI7Lw0bfwxUgA
 """
 
 import logging
 
-import py_utils
-
 # Default Cache Temperature. The page doesn't care which browser cache state
 # it is run on.
 ANY = 'any'
 # Emulates cold runs. Clears various caches and data with using tab.ClearCache()
 # and tab.ClearDataForOrigin().
 COLD = 'cold'
-# Emulates warm runs. Ensures that the page was visited at least once just
-# before the run.
+# Emulates warm runs. Ensures that the page was visited once before the run.
 WARM = 'warm'
+# Emulates hot runs. Ensures that the page was visited at least twice before
+# the run.
+HOT = 'hot'
 
 # These regacy states will be removed after chromium test scripts are adapted
 # to new states.
 PCV1_COLD = COLD
 PCV1_WARM = WARM
 
 class MarkTelemetryInternal(object):
 
   def __init__(self, browser, identifier):
     self.browser = browser
     self.identifier = identifier
 
   def __enter__(self):
     marker = 'telemetry.internal.%s.start' % self.identifier
     self.browser.tabs[0].ExecuteJavaScript(
         "console.time({{ marker }});", marker=marker)
     self.browser.tabs[0].ExecuteJavaScript(
         "console.timeEnd({{ marker }});", marker=marker)
     return self
 
   def __exit__(self, exception_type, exception_value, traceback):
     if exception_type:
       return True
-
     marker = 'telemetry.internal.%s.end' % self.identifier
     self.browser.tabs[0].ExecuteJavaScript(
         "console.time({{ marker }});", marker=marker)
     self.browser.tabs[0].ExecuteJavaScript(
         "console.timeEnd({{ marker }});", marker=marker)
     return True
 
-def ClearCache(browser):
+def ClearCacheAndData(browser, url):
   tab = browser.tabs[0]
   tab.ClearCache(force=True)
+  tab.ClearDataForOrigin(url)
 
-def WarmCache(page, browser):
-  with MarkTelemetryInternal(browser, 'warm_cache'):
+def WarmCache(page, browser, temperature):
+  with MarkTelemetryInternal(browser, 'warm_cache.%s' % temperature):
     tab = browser.tabs[0]
-    tab.Navigate(page.url)
-    py_utils.WaitFor(tab.HasReachedQuiescence, 60)
-    tab.WaitForDocumentReadyStateToBeComplete()
+    page.RunNavigateSteps(tab.action_runner)

[nednguyen, 2017/09/26 10:07:19]

wait, you mean page.RunNavigateSteps(page.url), right? You cannot navigate to
tab.action_runner, which is an object, not a url string.

[yukiy, 2017/09/26 10:36:25]

In loading.mobile, it will be PageCyclerStory.RunNavigateSteps(action_runner).
https://cs.chromium.org/chromium/src/tools/perf/page_sets/loading_mobile.py?d...
I changed it in other CL:
https://chromium-review.googlesource.com/c/chromium/src/+/647128

Also page.RunNavigateSteps(action_runner) is called in unittest.
https://cs.chromium.org/chromium/src/third_party/catapult/telemetry/telemetry...

[nednguyen, 2017/09/26 10:49:04]

ok

+    page.RunPageInteractions(tab.action_runner)
     tab.Navigate("about:blank")
     tab.WaitForDocumentReadyStateToBeComplete()
+    # Stop service worker after each cache warming to ensure service worker
+    # script evaluation will be executed again in next navigation.
+    tab.StopAllServiceWorkers()
 
 def EnsurePageCacheTemperature(page, browser, previous_page=None):
   temperature = page.cache_temperature
   logging.info('PageCacheTemperature: %s', temperature)
   if temperature == ANY:
     return
   if temperature == COLD:
     if previous_page is None:
       # DiskCache initialization is performed asynchronously on Chrome start-up.
       # Ensure that DiskCache is initialized before starting the measurement to
       # avoid performance skew.
       # This is done by navigating to an inexistent URL and then wait for the
       # navigation to complete.
       # TODO(kouhei) Consider moving this logic to PageCyclerStory
       with MarkTelemetryInternal(browser, 'ensure_diskcache'):
         tab = browser.tabs[0]
         tab.Navigate("http://does.not.exist")
         tab.WaitForDocumentReadyStateToBeComplete()
-    ClearCache(browser)
+    ClearCacheAndData(browser, page.url)
   elif temperature == WARM:
     if (previous_page is not None and
         previous_page.url == page.url and
-        (previous_page.cache_temperature == COLD or
-         previous_page.cache_temperature == WARM)):
+        previous_page.cache_temperature == COLD):
       if '#' in page.url:
         # Navigate to inexistent URL to avoid in-page hash navigation.
         # Note: Unlike PCv1, PCv2 iterates the same URL for different cache
         #       configurations. This may issue blink in-page hash navigations,
         #       which isn't intended here.
         with MarkTelemetryInternal(browser, 'avoid_double_hash_navigation'):
           tab = browser.tabs[0]
           tab.Navigate("http://does.not.exist")
           tab.WaitForDocumentReadyStateToBeComplete()
-      return
-    WarmCache(page, browser)
+      # Stop all service workers before running tests to measure the starting
+      # time of service worker too.
+      browser.tabs[0].StopAllServiceWorkers()
+    else:
+      ClearCacheAndData(browser, page.url)
+      WarmCache(page, browser, WARM)
+  elif temperature == HOT:
+    if (previous_page is not None and
+        previous_page.url == page.url and
+        previous_page.cache_temperature != ANY):
+      if previous_page.cache_temperature == COLD:
+        WarmCache(page, browser, HOT)
+      else:
+        if '#' in page.url:
+          # Navigate to inexistent URL to avoid in-page hash navigation.
+          # Note: Unlike PCv1, PCv2 iterates the same URL for different cache
+          #       configurations. This may issue blink in-page hash navigations,
+          #       which isn't intended here.
+          with MarkTelemetryInternal(browser, 'avoid_double_hash_navigation'):
+            tab = browser.tabs[0]
+            tab.Navigate("http://does.not.exist")
+            tab.WaitForDocumentReadyStateToBeComplete()
+        # Stop all service workers before running tests to measure the starting
+        # time of service worker too.
+        browser.tabs[0].StopAllServiceWorkers()

[falken, 2017/09/26 08:46:06]

I defer to kouhei and ned for the owner review. But this nested if/else plus
duplicated comments+code is a bit difficult to read. I'd suggest at least
splitting lines 116-127 into common helper function(s)?

[yukiy, 2017/09/26 09:01:32]

Sorry for confusing you, but it was discussed in other CL:
https://codereview.chromium.org/3013213002/

[nednguyen, 2017/09/26 10:07:19]

shimazu@: can you pick up the refactoring work after this? I agree with falken
is this code is too difficult to read. 

+    else:
+      ClearCacheAndData(browser, page.url)
+      WarmCache(page, browser, WARM)
+      WarmCache(page, browser, HOT)