Made Pixel tests functional. Captures tab content and compares to ref image

content/test/gpu/gpu_tests/pixel.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+from datetime import datetime
+import glob
+import os
+import re
+
 from telemetry import test
+from telemetry.core.backends import png_bitmap
 from telemetry.page import page_test
+from telemetry.page import test_expectations
 
+test_data_dir = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
+
+generated_data_dir = os.path.join(test_data_dir, 'generated')
+ref_image_dir = os.path.join(test_data_dir, 'gpu_reference')

[Ken Russell (switch to Gerrit), 2013/09/06 00:35:35]

These directories really need to be passed in from the outside. Could you follow
the pattern of the --generated-dir and --reference-dir arguments in the previous
harness in
http://build.chromium.org/p/chromium.gpu/builders/Linux%20Release%20%28NVIDIA...
?

+
+test_harness_script = r"""
+  var domAutomationController = {};
+
+  domAutomationController._succeeded = false;
+  domAutomationController._finished = false;
+
+  domAutomationController.setAutomationId = function(id) {}
+
+  domAutomationController.send = function(msg) {
+    domAutomationController._finished = true;
+
+    if(msg.toLowerCase() == "success") {
+      domAutomationController._succeeded = true;
+    } else {
+      domAutomationController._succeeded = false;
+    }
+  }
+
+  window.domAutomationController = domAutomationController;
+  console.log("Harness injected.");
+"""
 
 class PixelTestFailure(Exception):
   pass
 
+def _DidTestSucceed(tab):
+  return tab.EvaluateJavaScript('domAutomationController._succeeded')
 
 class PixelValidator(page_test.PageTest):
   def __init__(self):
     super(PixelValidator, self).__init__('ValidatePage')
 
+  def CustomizeBrowserOptions(self, options):
+    options.AppendExtraBrowserArg('--enable-gpu-benchmarking')
+
   def ValidatePage(self, page, tab, results):
-    # TODO(bajones): Grab screenshot, compare to reference.
-    # page.reference_image
-    pass
+    if not _DidTestSucceed(tab):
+      raise page_test.Failure("Page indicated a failure")
 
+    if not tab.screenshot_supported:
+      raise page_test.Failure("Browser does not support screenshot capture")
+
+    screenshot = tab.Screenshot(5)
+
+    if not screenshot:
+      raise page_test.Failure("Could not capture screenshot")
+
+    image_name = PixelValidator.UrlToImageName(page.url)
+
+    ref_png = PixelValidator.GetReferenceImage(image_name, page.revision,
+        screenshot)
+
+    # Test new snapshot against existing reference image
+    if not ref_png.IsEqual(screenshot, tolerance=2):
+      PixelValidator.WriteErrorImages(image_name, page.revision, screenshot,
+          ref_png)
+      raise page_test.Failure("Reference image did not match captured screen")
+
+  @staticmethod
+  def UrlToImageName(url):
+    image_name = re.sub(r'^(http|https|file)://(/*)', '', url)
+    image_name = re.sub(r'\.\./', '', image_name)
+    image_name = re.sub(r'(\.|/|-)', '_', image_name)
+    return image_name
+
+  @staticmethod
+  def DeleteOldReferenceImages(ref_image_path, cur_revision):
+    if not cur_revision:
+      return
+
+    old_revisions = glob.glob(ref_image_path + "__rev_*.png")
+    for rev_path in old_revisions:
+      m = re.match(r'^.*__rev_(\d+)\.png$', rev_path)
+      if m and int(m.group(1)) < cur_revision:
+        print 'Found deprecated reference image. Deleting rev ' + m.group(1)
+        os.remove(rev_path)
+
+  @staticmethod
+  def GetReferenceImage(image_name, cur_revision, screenshot):
+    if not cur_revision:
+      cur_revision = 0
+
+    image_path = os.path.join(ref_image_dir, image_name)
+
+    PixelValidator.DeleteOldReferenceImages(image_path, cur_revision)
+
+    image_path = image_path + '__rev_' + str(cur_revision) + '.png'
+
+    try:
+      ref_png = png_bitmap.PngBitmap.FromFile(image_path)
+    except IOError:
+      ref_png = None
+
+    if ref_png:
+      return ref_png
+
+    print 'Reference image not found. Writing tab contents as reference.'
+
+    PixelValidator.WriteImage(image_path, screenshot)
+    return screenshot
+
+  @staticmethod
+  def WriteErrorImages(image_name, cur_revision, screenshot, ref_png):
+    capture_time = datetime.now()
+    full_image_name = image_name + '__rev_' + str(cur_revision)
+    full_image_name = full_image_name + capture_time.strftime("__%b%d_%H_%M_%S")
+    full_image_name = full_image_name + '.png'
+
+    PixelValidator.WriteImage(
+        os.path.join(generated_data_dir, 'FAIL_' + full_image_name), screenshot)
+
+    diff_png = screenshot.Diff(ref_png)
+    PixelValidator.WriteImage(
+        os.path.join(generated_data_dir, 'DIFF_' + full_image_name), diff_png)
+
+  @staticmethod
+  def WriteImage(image_path, png_image):
+    output_dir = os.path.dirname(image_path)
+    if not os.path.exists(output_dir):
+      os.makedirs(output_dir)
+
+    png_image.WriteFile(image_path)
 
 class Pixel(test.Test):
   enabled = False
   test = PixelValidator
   page_set = 'page_sets/pixel_tests.json'
+
+  def CreateExpectations(self, page_set):
+    print page_set.pages
+    for page in page_set.pages:
+      print 'Injecting harness into ', page.url
+      page.script_to_evaluate_on_commit = test_harness_script
+
+    return test_expectations.TestExpectations()