Changed selenium tests to recover from test crashes/hangs. Divided perceptual...

tests/selenium/test_runner.py

+#!/usr/bin/python2.4
+# Copyright 2009, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+"""Test runners and associated classes.
+
+Each test runner has its own thread, which attempts to perform a given test.
+If a test hangs, the test runner can be aborted or exited.
+
+"""
+
+import os
+import sys
+
+import socket
+import subprocess
+import threading
+import time
+import unittest
+import gflags
+import selenium
+import selenium_constants
+import Queue
+import thread
+import copy
+
+class StringBuffer:
+  """Primitive string buffer.
+
+  Members:
+    data: the contents of the buffer
+  """
+  def __init__(self):
+    self.data = ""
+  def write(self, data):
+    self.data += str(data)
+  def writeln(self, data=None):
+    if data is not None:
+      self.write(data)
+    self.write("\n")
+  def get(self):
+    get_data = self.data
+    self.data = ""
+    return get_data
+
+class TestResult(unittest.TestResult):
+  """A specialized class that prints formatted text results to a stream.
+
+  """
+  separator1 = "=" * 70
+  separator2 = "-" * 70
+
+  def __init__(self, stream, browser):
+    unittest.TestResult.__init__(self)
+    self.stream = stream
+    # Dictionary of start times
+    self.start_times = {}
+    # Dictionary of results
+    self.results = {}
+    self.browser = browser
+
+  def getDescription(self, test):
+    """Gets description of test."""
+    return test.shortDescription() or str(test)
+
+  def startTest(self, test):
+    """Starts test."""
+    # Records the start time
+    self.start_times[test] = time.time()
+    # Default testresult if success not called
+    self.results[test] = "FAIL"
+    unittest.TestResult.startTest(self, test)
+    self.stream.writeln()
+    self.stream.writeln(self.separator2)
+    self.stream.write(self.getDescription(test))
+    self.stream.writeln(" ... ")
+
+  def stopTest(self, test):
+    """Called when test is ended."""
+    time_taken = time.time() - self.start_times[test]
+    result = self.results[test]
+    self.stream.writeln("SELENIUMRESULT %s <%s> [%.3fs]: %s"
+                        % (test, self.browser, time_taken, result))
+
+  def addSuccess(self, test):
+    """Adds success result to TestResult."""
+    unittest.TestResult.addSuccess(self, test)
+    self.results[test] = "PASS"
+
+  def addError(self, test, err):
+    """Adds error result to TestResult."""
+    unittest.TestResult.addError(self, test, err)
+    self.results[test] = "FAIL"
+
+  def addFailure(self, test, err):
+    """Adds failure result to TestResult."""
+    unittest.TestResult.addFailure(self, test, err)
+    self.results[test] = "FAIL"
+
+  def noResponse(self, test):
+    """Configures the result for a test that did not respond."""
+    self.results[test] = "FAIL"
+    self.testsRun += 1
+    self.errors.append("No response from test")
+
+    self.stream.writeln()
+    self.stream.writeln(self.separator2)
+    self.stream.write(self.getDescription(test))
+    self.stream.writeln(" ... ")
+    self.stream.writeln("SELENIUMRESULT %s <%s> [?s]: FAIL (HUNG?)"
+                        % (test, self.browser))
+    self.stream.writeln()
+
+  def printErrors(self):
+    """Prints all errors and failures."""
+    self.stream.writeln()
+    self.printErrorList("ERROR", self.errors)
+    self.printErrorList("FAIL", self.failures)
+
+  def printErrorList(self, flavour, errors):
+    """Prints a given list of errors."""
+    for test, err in errors:
+      self.stream.writeln("%s:" % flavour)
+      self.stream.writeln("%s" % err)
+
+  def printAll(self, stream):
+    """Prints the entire stream to the given stream."""
+    stream.write(self.stream.data)
+
+  def merge(self, result):
+    """Merges the given result into this resultl."""
+    self.testsRun += result.testsRun
+    for key, entry in result.results.iteritems():
+      self.results[key] = entry
+    for error in result.errors:
+      self.errors.append(error)
+    for failure in result.failures:
+      self.failures.append(failure)
+    self.stream.write(result.stream)
+
+
+class TestRunnerThread(threading.Thread):
+  """Abstract test runner class.  Launches its own thread for running tests.
+  Formats test results.
+
+  Members:
+    completely_done_event: event that occurs just before thread exits.
+    test: the currently running test.
+    browser: selenium_name of browser that will be tested.
+  """
+  def __init__(self):
+    threading.Thread.__init__(self)
+    # This thread is a daemon so that the program can exit even if the
+    # thread has not finished.
+    self.setDaemon(True)
+    self.completely_done_event = threading.Event()
+    self.test_copy = None
+    self.browser = "default_browser"
+
+  def IsCompletelyDone(self):
+    """Returns true if this test runner is completely done."""
+    return self.completely_done_event.isSet()
+
+  def run(self):
+    pass
+    
+  def SetBrowser(self, browser):
+    """Sets the browser name."""
+    self.browser = browser
+
+  def GetNoResponseResult(self):
+    """Returns a generic no response result for last test."""
+    result = TestResult(StringBuffer(), self.browser)
+    result.noResponse(self.test)
+    return result
+
+  def RunTest(self, test):
+    "Run the given test case or test suite."
+    self.test = test
+
+    stream = StringBuffer()
+    result = TestResult(stream, self.browser)
+    startTime = time.time()
+    test(result)
+    stopTime = time.time()
+    timeTaken = stopTime - startTime
+    result.printErrors()
+    run = result.testsRun
+    stream.writeln("Took %.2fs" % timeTaken)
+    stream.writeln()
+    return result
+  
+
+class PDiffTestRunner(TestRunnerThread):
+  """Test runner for Perceptual Diff tests. Polls a test queue and launches
+  given tests. Adds result to given queue.
+
+  Members:
+    pdiff_queue: list of tests to run, when they arrive.
+    result_queue: queue of our tests results.
+    browser: selenium name of browser to be tested.
+    end_testing_event: event that occurs when we are guaranteed no more tests
+      will be added to the queue.
+  """
+  def __init__(self, pdiff_queue, result_queue, browser):
+    TestRunnerThread.__init__(self)
+    self.pdiff_queue = pdiff_queue
+    self.result_queue = result_queue
+    self.browser = browser
+    
+    self.end_testing_event = threading.Event()
+
+  def EndTesting(self):
+    """Called to notify thread that no more tests will be added to the test
+    queue."""
+    self.end_testing_event.set()
+
+  def run(self):
+    while True:
+      try:
+        test = self.pdiff_queue.get_nowait()
+
+        result = self.RunTest(test)
+        
+        self.result_queue.put(result)
+        
+      except Queue.Empty:
+        if self.end_testing_event.isSet() and self.pdiff_queue.empty():
+          break
+        else:
+          time.sleep(1)
+
+    self.completely_done_event.set()
+    
+
+class SeleniumTestRunner(TestRunnerThread):
+  """Test runner for Selenium tests. Takes a test from a test queue and launches
+  it. Tries to handle hung/crashed tests gracefully.
+
+  Members:
+    testing_event: event that occurs when the runner is testing.
+    finished_event: event that occurs when thread has finished testing and
+      before it starts its next test.
+    can_continue_lock: lock for |can_continue|.
+    can_continue: is True when main thread permits the test runner to continue.
+    sel_builder: builder that constructs new selenium sessions, as needed.
+    browser: selenium name of browser to be tested.
+    session: current selenium session being used in tests, can be None.
+    test_queue: queue of tests to run.
+    pdiff_queue: queue of perceptual diff tests to run. We add a perceptual
+      diff test to the queue when the related selenium test passes.
+    deadline: absolute time of when the test should be done.
+  """
+  def __init__(self, sel_builder, browser, test_queue, pdiff_queue):
+    TestRunnerThread.__init__(self)
+
+    # Synchronization.
+    self.testing_event = threading.Event()
+    self.finished_event = threading.Event()
+    self.can_continue_lock = threading.Lock()
+    self.can_continue = False
+    
+    # Selenium variables.
+    self.sel_builder = sel_builder
+    self.browser = browser
+
+    # Test variables.
+    self.test_queue = test_queue
+    self.pdiff_queue = pdiff_queue
+    
+    self.deadline = 0
+    
+  def IsPastDeadline(self):
+    if time.time() > self.deadline:
+      return True
+    return False
+
+  def IsTesting(self):
+    return self.testing_event.isSet()
+
+  def DidFinishTest(self):
+    return self.finished_event.isSet()
+  
+  def Continue(self):
+    """Signals to thread to continue testing.
+
+    Returns:
+      result: the result for the recently finished test.
+    """
+
+    self.finished_event.clear()
+    
+    self.can_continue_lock.acquire()
+    self.can_continue = True
+    result = self.result
+    self.can_continue_lock.release()
+    
+    return result
+    
+  def AbortTest(self):
+    self._StopSession()
+    self._StartSession()
+    
+  def _StartSession(self):
+    self.session = self.sel_builder.NewSeleniumSession(self.browser)
+    # Copy the session so we can shut down safely on a different thread.
+    self.shutdown_session = copy.deepcopy(self.session)
+
+  def _StopSession(self):
+    if self.session is not None:
+      self.session = None
+      try:
+        # This can cause an exception on some browsers.
+        # Silenly disregard the exception.
+        self.shutdown_session.stop()
+      except:
+        pass
+  
+  def run(self):
+    self._StartSession()
+    
+    while not self.test_queue.empty():
+      try:
+        # Grab test from queue.
+        test_obj = self.test_queue.get_nowait()
+        if type(test_obj) == tuple:
+          test = test_obj[0]
+          pdiff_test = test_obj[1]
+        else:
+          test = test_obj
+          pdiff_test = None
+
+        self.can_continue = False
+
+        # Deadline is the time to load page timeout plus a constant.
+        self.deadline = (time.time() + (test.GetTestTimeout() / 1000.0) +
+                         selenium_constants.MAX_SELENIUM_TEST_TIME)
+        # Supply test with necessary selenium session.
+        test.SetSession(self.session)
+
+        # Run test.
+        self.testing_event.set()
+        self.result = self.RunTest(test)
+
+        if time.time() > self.deadline:
+          self.result = self.GetNoResponseResult()
+        
+        self.testing_event.clear()
+        self.finished_event.set()
+
+        # Wait for instruction from the main thread.
+        while True:
+          self.can_continue_lock.acquire()
+          can_continue = self.can_continue
+          self.can_continue_lock.release()
+          if can_continue:
+            break
+          time.sleep(.5)        
+
+        if self.pdiff_queue is not None and pdiff_test is not None:
+          if self.result.wasSuccessful():
+            # Add the dependent perceptual diff test.
+            self.pdiff_queue.put(pdiff_test)       
+        
+      except Queue.Empty:
+        break
+
+    self._StopSession()      
+    self.completely_done_event.set()
+ 
+