CNS seek tests for <video>.

chrome/test/functional/media/media_seek_perf.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Seek performance testing for <video>.
+
+Calculates the short and long seek times for different video formats on
+different network constraints.
+"""
+
+import itertools
+import logging
+import os
+import Queue
+import string
+import threading
+
+import pyauto_media
+import pyauto
+import pyauto_utils
+
+import cns_test_base
+from cns_test_base import CNSTestBase
+
+# Number of threads to use during testing.

[DaleCurtis, 2012/04/10 23:00:06]

Lots of duplicated defines here. Should refactor against
media_constrained_network_perf.

+_TEST_THREADS = 3
+
+# HTML test path; relative to src/chrome/test/data.
+_TEST_HTML_PATH = os.path.join('media', 'html', 'media_seek.html')
+
+# The media files used for testing.
+# Path under CNS root folder (pyauto_private/media).
+_TEST_VIDEOS = [os.path.join('crowd', name) for name in
+    ['crowd1080.webm', 'crowd720.webm', 'crowd480.webm', 'crowd360.webm']]
+
+# Constraints to run tests on.
+_TESTS_TO_RUN = {
+    '512kbps_105ms': [512, 105, 0],
+    'Wifi_1Mbps_60ms': [1024, 60, 0],
+    'DSL_1.5Mbps_50ms': [1541, 50, 0],
+    'Cable_5Mbps_28ms': [5120, 28, 0],
+    'NoConstraints': [0, 0, 0]
+}
+
+
+class TestWorker(threading.Thread):

[DaleCurtis, 2012/04/10 23:00:06]

This worker code should probably be refactored/abstracted out a bit if multiple
tests are going to use it.

Even more awesome would be building a general framework in PyAuto for threaded
tab testing.

+  """Worker thread.  For each queue entry: opens tab, runs test, closes tab."""
+
+  # Atomic, monotonically increasing task identifier.  Used to ID tabs.
+  _task_id = itertools.count()
+
+  def __init__(self, pyauto_test, tasks, automation_lock, url):
+    """Sets up TestWorker class variables.
+
+    Args:
+      pyauto_test: Reference to a pyauto.PyUITest instance.
+      tasks: Queue containing (settings, name) tuples.
+      automation_lock: Global automation lock for pyauto calls.
+      url: File URL to HTML/JavaScript test code.
+    """
+    threading.Thread.__init__(self)
+    self._tasks = tasks
+    self._automation_lock = automation_lock
+    self._pyauto = pyauto_test
+    self._url = url
+    self._uncached_seeks = []
+    self._cached_seeks = []
+    self._error_msg = ''
+    self.start()
+
+  def _FindTabLocked(self, url):
+    """Returns the tab index for the tab belonging to this url.
+
+    self._automation_lock must be owned by caller.
+    """
+    for tab in self._pyauto.GetBrowserInfo()['windows'][0]['tabs']:
+      if tab['url'] == url:
+        return tab['index']
+
+  def _EndTest(self, unique_url):
+    """Checks if the page has variable value ready or if an error has occured.
+
+    The varaible value must be set to < 0 pre-run.
+
+    Args:
+      var_name: The variable name to check the metric for.
+      unique_url: The url of the page to check for the variable's metric.
+
+    Returns:
+      True is the var_name value is >=0 or if an error_msg exists.
+    """
+    with self._automation_lock:
+      tab = self._FindTabLocked(unique_url)
+      self._error_msg = self._pyauto.GetDOMValue('errorMsg', tab_index=tab)
+      end_test = self._pyauto.GetDOMValue('endTest', tab_index=tab)
+      if end_test:
+        self._uncached_seeks = [
+            float(value) for value in
+            self._pyauto.GetDOMValue("longSeeks.join(',')",
+                                     tab_index=tab).split(',')]
+        self._cached_seeks = [
+            float(value) for value in
+            self._pyauto.GetDOMValue("shortSeeks.join(',')",
+                                     tab_index=tab).split(',')]
+    return self._error_msg or end_test
+
+  def run(self):
+    """Opens tab, starts HTML test, and records metrics for each queue entry.
+
+    No exception handling is done to make sure the main thread exits properly
+    during Chrome crashes or other failures.  Doing otherwise has the potential
+    to leave the CNS server running in the background.
+
+    For a clean shutdown, put the magic exit value (None, None) in the queue.
+    """
+    # Make the test URL unique so we can figure out our tab index later.
+    unique_url = '%s?%d' % (self._url, TestWorker._task_id.next())
+    with self._automation_lock:
+      self._pyauto.AppendTab(pyauto.GURL(unique_url))
+
+    while True:
+      series_name, settings, file_name = self._tasks.get()
+
+      # Check for magic exit values.
+      if (series_name, settings, file_name) == (None, None, None):
+        break
+
+      video_url = cns_test_base.GetFileURL(
+          file_name, bandwidth=settings[0], latency=settings[1],
+          loss=settings[2], new_port=True)
+
+      self._uncached_seeks = []
+      self._cached_seeks = []
+      self._error_msg = ''
+      # Start the test!
+      with self._automation_lock:
+        self._pyauto.CallJavascriptFunc(
+            'startTest', [video_url],
+            tab_index=self._FindTabLocked(unique_url))
+      logging.debug('Running perf test for %s.', video_url)
+
+      self._pyauto.WaitUntil(
+          self._EndTest, args=[unique_url], retry_sleep=1, timeout=500,
+          debug=False)
+
+      if self._error_msg:
+        logging.error('Error while running the test: %s' % self._error_msg)
+      else:
+        graph_name = series_name +'_' + os.path.basename(file_name)
+        pyauto_utils.PrintPerfResult('seek', 'uncached_' + graph_name,
+                                     self._uncached_seeks, 'ms')
+        pyauto_utils.PrintPerfResult('seek', 'cached_' + graph_name,
+                                     self._cached_seeks, 'ms')
+
+    # Close the tab.
+    with self._automation_lock:
+      self._pyauto.GetBrowserWindow(0).GetTab(
+          self._FindTabLocked(unique_url)).Close(True)
+
+    self._tasks.task_done()
+
+
+class MediaSeekPerfTest(CNSTestBase, pyauto.PyUITest):
+  """PyAuto test container.  See file doc string for more information."""
+
+  def testMediaSeekPerformance(self):
+    """Launches HTML test which plays each video and records seek stats."""
+    # Convert relative test path into an absolute path.
+    test_url = self.GetFileURLForDataPath(_TEST_HTML_PATH)
+
+    # PyAuto doesn't support threads, so we synchronize all automation calls.
+    automation_lock = threading.Lock()
+
+    # Spin up worker threads.

[DaleCurtis, 2012/04/10 23:00:06]

No dummy test necessary anymore?

+    tasks = Queue.Queue()
+    for file_name in _TEST_VIDEOS:
+      for series_name, settings in _TESTS_TO_RUN.iteritems():
+        logging.debug('Add test: %s\tSettings: %s\tMedia: %s', series_name,
+                      settings, file_name)
+        tasks.put((series_name, settings, file_name))
+
+    # Add shutdown magic to end of queue.
+    for thread in xrange(_TEST_THREADS):
+      tasks.put((None, None, None))
+
+    threads = []
+    for _ in xrange(_TEST_THREADS):
+      threads.append(TestWorker(self, tasks, automation_lock, test_url))
+
+    # Wait for threads to exit, gracefully or otherwise.
+    for thread in threads:
+      thread.join()
+
+
+if __name__ == '__main__':
+  pyauto_media.Main()