Introduce new PyAuto test to measure EPT, TTP metrics.

chrome/test/functional/media/media_cn_perf.py

+#!/usr/bin/env python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Records metrics on playing media under constrained network conditions.
+
+Spins up a Constrained Network Server (CNS) and runs through a test matrix of
+bandwidth, latency, and packet loss settings. Each run records a
+time-to-playback (TTP) and extra-play-time (EPT) metric in a format consumable
+by the Chromium perf bots.
+
+See design document "Testing Video Performance Over Constrained Networks" for
+more information -- http://goo.gl/EdAWU
+
+Since even a small number of different settings yields a large test matrix, the
+design is threaded... however PyAuto is not, so a global lock is used when calls
+into PyAuto are necessary. The number of threads can be set by _TEST_THREADS.
+
+The CNS code is located under: <root>/src/media/tools/constrained_network_server
+"""
+
+import itertools
+import logging
+import os
+import Queue
+import signal
+import subprocess
+import threading
+
+import pyauto_media
+import pyauto
+import pyauto_paths
+import pyauto_utils
+
+
+# Settings for each network constraint.
+_BANDWIDTH_SETTINGS_KBPS = {'Off': 0, 'Low': 256, 'Medium': 2000, 'High': 5000}
+_LATENCY_SETTINGS_MS = {'Off': 0, 'Low': 43, 'Medium': 105, 'High': 180}
+_PACKET_LOSS_SETTINGS_PERCENT = {'Off': 0, 'Medium': 2, 'High': 5}
+
+# Test constraints are all possible combination of the above settings. Each

[Nirnimesh, 2011/12/06 22:15:05]

Leave 2 blank spaces after every period, ie before 'Each'
(See Comments section in http://www.python.org/dev/peps/pep-0008/)

[DaleCurtis, 2011/12/06 23:42:54]

Ugh, if you insist. In protest, I'd like to point out that it's inconsistent
with the majority of the PyAuto code and it's a deprecated style:

http://www.slate.com/articles/technology/technology/2011/01/space_invaders.html
http://en.wikipedia.org/wiki/Space_(punctuation)#Spaces_between_sentences

[Nirnimesh, 2011/12/06 23:58:36]

I don't like it either. All my initial pyauto code has one space after period,
but I was pointed to the style guide
(http://www.chromium.org/developers/coding-style which points to PEP-8) which
specifically asks for 2 spaces. So I use 2 spaces for all new python code.

However, I don't insist.

+# tuple must be of the form (Bandwidth, Latency, Packet Loss).

[Nirnimesh, 2011/12/06 22:15:05]

s/tuple/Tuple

[DaleCurtis, 2011/12/06 23:42:54]

Tuple is not a proper noun and this is a continuation of the previous line.

+_TEST_CONSTRAINTS = itertools.product(
+    _BANDWIDTH_SETTINGS_KBPS.values(),
+    _LATENCY_SETTINGS_MS.values(),
+    _PACKET_LOSS_SETTINGS_PERCENT.values())
+
+_TEST_CONSTRAINT_NAMES = itertools.product(
+    _BANDWIDTH_SETTINGS_KBPS.keys(),
+    _LATENCY_SETTINGS_MS.keys(),
+    _PACKET_LOSS_SETTINGS_PERCENT.keys())
+
+# HTML test path; relative to src/chrome/test/data. Loads a test video and
+# records metrics in JavaScript.
+_TEST_HTML_PATH = os.path.join('media', 'html', 'media_cn.html')
+
+# Number of threads to use during testing. TODO(dalecurtis): Should be set on

[Nirnimesh, 2011/12/06 22:15:05]

move TODO to next line

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+# the command line.
+_TEST_THREADS = 3
+
+# File name of video to collect metrics for. TODO(dalecurtis): Should be set on

[Nirnimesh, 2011/12/06 22:15:05]

move TODO to next line

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+# the command line.
+_TEST_VIDEO = 'dancing.webm'
+
+# Path to CNS executable relative to source root.
+_CNS_PATH = os.path.join(
+    'media', 'tools', 'constrained_network_server', 'cns.py')
+
+# Port to start the CNS on.
+_CNS_PORT = 9000
+
+# Base CNS URL, only requires & separated parameter names appended.
+_CNS_BASE_URL = 'http://127.0.0.1:%d/ServeConstrained?' % _CNS_PORT
+
+
+class TestWorker(threading.Thread):
+  """Worker thread. For each queue entry: opens tab, runs test, closes tab."""
+
+  # Atomic, monotonically increasing task identifier. Used to ID tabs.
+  _task_id = itertools.count()
+
+  def __init__(self, pyauto_test, tasks, automation_lock, url):
+    """Sets up TestWorker class variables.
+
+    Args:
+      pyauto_test: Reference to a pyauto.PyUITest instance.
+      tasks: Queue containing (settings, name) tuples.
+      automation_lock: Global automation lock for pyauto calls.
+      url: File URL to HTML/JavaScript test code.
+    """
+    threading.Thread.__init__(self)
+    self._tasks = tasks
+    self._automation_lock = automation_lock
+    self._pyauto = pyauto_test
+    self._url = url
+    self.start()
+
+  def _FindTab(self, url):
+    """Returns the tab index for the tab belonging to this thread."""

[Nirnimesh, 2011/12/06 22:15:05]

s/thread/url/ ?

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+    for tab in self._pyauto.GetBrowserInfo()['windows'][0]['tabs']:
+      if tab['url'] == url:
+        return tab['index']
+
+  def _HaveMetrics(self, unique_url):
+    """Returns if metrics are ready or not. Set self.{_ept,_ttp} < 0 pre-run."""
+    with self._automation_lock:
+      tab = self._FindTab(unique_url)
+
+      if self._ept < 0:
+        self._ept = self._pyauto.GetDOMValue('extra_play_time', tab_index=tab)
+      if self._ttp < 0:
+        self._ttp = self._pyauto.GetDOMValue('time_to_playback', tab_index=tab)
+    return self._ept >= 0 and self._ttp >= 0
+
+  def run(self):
+    """Opens tab, starts HTML test, and records metrics for each queue entry.
+
+    No exception handling is done to make sure the main thread exits properly
+    during Chrome crashes or other failures. Doing otherwise has the potential
+    to leave the CNS server running in the background.
+
+    For a clean shutdown, put the magic exit value (None, None) in the queue.
+    """
+    while True:
+      settings, name = self._tasks.get()
+
+      # Listen for magic exit values.

[Nirnimesh, 2011/12/06 22:15:05]

s/Listen/Check/

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+      if (settings, name) == (None, None):
+        break
+
+      # Build video source URL. Values <= 0 mean the setting is disabled.
+      video_url = [_CNS_BASE_URL, 'f=' + _TEST_VIDEO]
+      if settings[0] > 0:
+        video_url.append('bandwidth=%d' % settings[0])
+      if settings[1] > 0:
+        video_url.append('latency=%d' % settings[1])
+      if settings[2] > 0:
+        video_url.append('loss=%d' % settings[2])
+      video_url = '&'.join(video_url)
+
+      # Make the test URL unique so we can figure out our tab index later.

[Nirnimesh, 2011/12/06 22:15:05]

why not save the tab index?

[DaleCurtis, 2011/12/06 23:42:54]

When we talked via chat you said the tab_index changes when tabs are closed. Is
this not correct? Another thread may have closed a tab in between critical
sections.

[Nirnimesh, 2011/12/06 23:58:36]

I see. Yes, that's correct.

To that end, why do you need to close the tabs?

[DaleCurtis, 2011/12/07 01:30:05]

I could probably get away with not doing so, but with 48 potential tabs open, it
seemed risky when it's trivial to protect against.

+      unique_url = '%s?%d' % (self._url, TestWorker._task_id.next())
+
+      # Start the test!
+      with self._automation_lock:
+        self._pyauto.AppendTab(pyauto.GURL(unique_url))
+        self._pyauto.CallJavascriptFunc(
+            'startTest', [video_url], tab_index=self._FindTab(unique_url))
+
+      # Wait until the necessary metrics have been collected. Okay to not lock

[Nirnimesh, 2011/12/06 22:15:05]

2 spaces after .

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+      # here since pyauto.WaitUntil doesn't call into Chrome.

[Nirnimesh, 2011/12/06 22:15:05]

but _HaveMetrics calls into chrome..

[DaleCurtis, 2011/12/06 23:42:54]

...and makes those calls with the lock.

+      self._ept = self._ttp = -1
+      self._pyauto.WaitUntil(
+          self._HaveMetrics, args=[unique_url], retry_sleep=2)
+
+      # Record results. TODO(dalecurtis): Support reference builds.

[Nirnimesh, 2011/12/06 22:15:05]

move todo to next line

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+      series_name = ''.join(name)
+      pyauto_utils.PrintPerfResult('ept', series_name, self._ept, '%')
+      pyauto_utils.PrintPerfResult('ttp', series_name, self._ttp, 'ms')
+
+      # Close the tab.
+      with self._automation_lock:
+        self._pyauto.GetBrowserWindow(0).GetTab(
+            self._FindTab(unique_url)).Close(True)
+
+      # TODO(dalecurtis): Check results for regressions.
+      self._tasks.task_done()
+
+
+class MediaConstrainedNetworkPerfTest(pyauto.PyUITest):
+  """PyAuto test container. See file doc string for more information."""
+
+  def setUp(self):
+    """Starts the Constrained Network Server (CNS)."""
+    cmd = ['python', os.path.join(pyauto_paths.GetSourceDir(), _CNS_PATH),

[Nirnimesh, 2011/12/06 22:15:05]

s/python/sys.executable

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+           '--port', str(_CNS_PORT),
+           '--interface', 'lo',
+           '--www-root', os.path.join(self.DataDir(), 'media')]
+    process = subprocess.Popen(cmd, stderr=subprocess.PIPE)
+
+    # Wait for server to start up.
+    line = True
+    while line:
+      line = process.stderr.readline()
+      if 'STARTED' in line:
+        self._server_pid = process.pid
+        pyauto.PyUITest.setUp(self)
+        return
+    self.fail('Failed to start CNS.')
+
+  def tearDown(self):
+    """Stops the Constrained Network Server (CNS)."""
+    self.Kill(self._server_pid)
+    pyauto.PyUITest.tearDown(self)

[Nirnimesh, 2011/12/06 22:15:05]

destruct in the opposite order.
ie tearDown before CNS Kill

[DaleCurtis, 2011/12/06 23:42:54]

Done.

+
+  def testConstrainedNetworkPerf(self):
+    """Starts CNS, spins up worker threads to run through _TEST_CONSTRAINTS."""
+    # Convert relative test path into an absolute path.
+    test_url = self.GetFileURLForDataPath(_TEST_HTML_PATH)
+
+    # PyAuto doesn't support threads, so we synchronize all automation calls.

[Nirnimesh, 2011/12/06 22:15:05]

s/synchronize/guard around/ ?

[DaleCurtis, 2011/12/06 23:42:54]

Synchronize works fine:
http://en.wikipedia.org/wiki/Synchronization_(computer_science)#Thread_or_pro...

+    automation_lock = threading.Lock()
+
+    # Spin up worker threads.
+    tasks = Queue.Queue()
+    threads = []
+    for _ in xrange(_TEST_THREADS):
+      threads.append(TestWorker(self, tasks, automation_lock, test_url))

[Nirnimesh, 2011/12/06 22:15:05]

Won't having multiple threads affect the numbers you get?

[DaleCurtis, 2011/12/06 23:42:54]

Not in the limited testing I've done thus far. We're limiting network traffic
enough that we should be fine with low numbers of threads. Once the CNS portion
is completed I'll do some more experiments. Worst case we just set threads = 1.

+
+    for settings, name in zip(_TEST_CONSTRAINTS, _TEST_CONSTRAINT_NAMES):
+      tasks.put((settings, name))
+
+    # Add shutdown magic to end of queue.
+    for thread in threads:
+      tasks.put((None, None))
+
+    # Wait for threads to exit, gracefully or otherwise.
+    for thread in threads:
+      thread.join()
+
+
+if __name__ == '__main__':
+  # TODO(dalecurtis): Process command line parameters here.
+  pyauto_media.Main()