Adding Netflix performance measurement tests for dropped frames and CPU extrapolation

functional/perf.py

 #!/usr/bin/python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Basic pyauto performance tests.
 
 For tests that need to be run for multiple iterations (e.g., so that average
 and standard deviation values can be reported), the default number of iterations
 run for each of these tests is specified by |_DEFAULT_NUM_ITERATIONS|.
@@ skipping 27 matching lines @@
 import urllib2
 import urlparse
 
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 
 import perf_snapshot
 import pyauto_utils
 import test_utils
 from youtube import YoutubeTestHelper
+from netflix import NetflixTestHelper

[dennis_jeffrey, 2011/11/21 22:45:10]

Move this up right before line 44, so that the application-specific helpers are
imported in alphabetical order ('netflix' comes before 'perf_snapshot').

[rohitbm, 2011/11/22 00:57:09]

Done.

 
 
 class BasePerfTest(pyauto.PyUITest):
   """Base class for performance tests."""
 
   _DEFAULT_NUM_ITERATIONS = 50
   _DEFAULT_MAX_TIMEOUT_COUNT = 10
   _PERF_OUTPUT_MARKER_PRE = '_PERF_PRE_'
   _PERF_OUTPUT_MARKER_POST = '_PERF_POST_'
 
@@ skipping 472 matching lines @@
     def _RunSingleDocsTabOpen():
       self._AppendTab('http://docs.google.com')
       self.assertTrue(self.WaitUntil(_SubstringExistsOnPage, timeout=120,
                                      expect_retval='true', retry_sleep=0.10),
                       msg='Timed out waiting for expected Docs string.')
 
     self._LoginToGoogleAccount()
     self._RunNewTabTest('NewTabDocs', _RunSingleDocsTabOpen)
 
 
+class NetflixPerfTest(BasePerfTest, NetflixTestHelper):
+  """Test Netflix video performance."""
+
+  def __init__(self, methodName='runTest', **kwargs):
+    pyauto.PyUITest.__init__(self, methodName, **kwargs)
+    NetflixTestHelper.__init__(self, self)
+
+  def tearDown(self):
+    self._SignOut()
+    pyauto.PyUITest.tearDown(self) 
+
+  def testNetflixDroppedFrames(self):
+    """Measures the Netflix video dropped frames. Runs for 60 secs."""

[dennis_jeffrey, 2011/11/21 22:45:10]

Clarify that this test measures the dropped frames per second (not the total
number of dropped frames).

[rohitbm, 2011/11/22 00:57:09]

Done.

+    self._LoginAndStartPlaying()
+    # Ignore first 10 seconds of video playing so we get smooth videoplayback.
+    time.sleep(10)
+    init_dropped_frames = self.GetVideoDroppedFrames()
+    dropped_frames = []
+    prev_dropped_frames = 0
+    for _ in xrange(60):
+      # Ignoring initial dropped frames of first 10 seconds.
+      total_dropped_frames =  self.GetVideoDroppedFrames() - init_dropped_frames
+      dropped_frames.append(total_dropped_frames - prev_dropped_frames)
+      prev_dropped_frames = total_dropped_frames
+      # Play the video for some time.
+      time.sleep(1)
+    self._PrintSummaryResults('NetflixDroppedFrames', dropped_frames, 'frames')
+
+  def testNetflixCPU(self):
+    """Measures the Netflix video CPU usage. Runs for 60 seconds."""
+    self._LoginAndStartPlaying()
+    # Ignore first 10 seconds of video playing so we get smooth videoplayback.
+    time.sleep(10)
+    init_dropped_frames = self.GetVideoDroppedFrames()
+    init_video_frames = self.GetVideoFrames()
+    cpu_usage_start = self._GetCPUUsage()
+    total_shown_frames = 0
+    # Play the video for some time.
+    time.sleep(60)
+    total_video_frames = self.GetVideoFrames() - init_video_frames
+    total_dropped_frames = self.GetVideoDroppedFrames() - init_dropped_frames
+    cpu_usage_end = self._GetCPUUsage()
+    fraction_non_idle_time =\

[dennis_jeffrey, 2011/11/21 22:45:10]

nit: put a space right before the \

[rohitbm, 2011/11/22 00:57:09]

Done.

+        self._GetFractionNonIdleCPUTime(cpu_usage_start, cpu_usage_end)
+    # Counting extrapolation for utilization to play the video.
+    extrapolation_value = fraction_non_idle_time *\

[dennis_jeffrey, 2011/11/21 22:45:10]

nit: put a space right before the \

[rohitbm, 2011/11/22 00:57:09]

Done.

+        (total_video_frames + total_dropped_frames)/total_video_frames

[dennis_jeffrey, 2011/11/21 22:45:10]

add spaces right before and right after the /

[rohitbm, 2011/11/22 00:57:09]

Done.

+    logging.info('Netflix CPU extrapolation: %.2f' % extrapolation_value)
+    self._OutputPerfGraphValue('extrapolation_NetflixCPUExtrapolation',
+                               extrapolation_value)
+
+
 class YoutubePerfTest(BasePerfTest, YoutubeTestHelper):
   """Test Youtube video performance."""
 
   def __init__(self, methodName='runTest', **kwargs):
     pyauto.PyUITest.__init__(self, methodName, **kwargs)
     YoutubeTestHelper.__init__(self, self)
 
   def _VerifyVideoTotalBytes(self):
     """Returns true if video total bytes information is available."""
     return self.GetVideoTotalBytes() > 0
@@ skipping 21 matching lines @@
     self.PauseVideo()
     # Wait for the video to finish loading.
     while total_video_bytes > loaded_video_bytes:
       loaded_video_bytes = self.GetVideoLoadedBytes()
       time.sleep(1)
     self.PlayVideo()
     # Ignore first 10 seconds of video playing so we get smooth videoplayback.
     time.sleep(10)
 
   def testYoutubeDroppedFrames(self):
     """Measures the Youtube video dropped frames. Runs for 60 secs."""

[dennis_jeffrey, 2011/11/21 22:45:10]

Clarify that this test measures the dropped frames per second (not the total
number of dropped frames).

[rohitbm, 2011/11/22 00:57:09]

Done.

     self.StartVideoForPerformance()
     init_dropped_frames = self.GetVideoDroppedFrames()
     total_dropped_frames = 0
     dropped_fps = []
     for _ in xrange(60):
       frames = self.GetVideoDroppedFrames() - init_dropped_frames
       current_dropped_frames = frames - total_dropped_frames 
       dropped_fps.append(current_dropped_frames)
       total_dropped_frames = frames
       # Play the video for some time
@@ skipping 752 matching lines @@
     """Identifies the port number to which the server is currently bound.
 
     Returns:
       The numeric port number to which the server is currently bound.
     """
     return self._server.server_address[1]
 
 
 if __name__ == '__main__':
   pyauto_functional.Main()