Sends test results to new perf dashboard

scripts/slave/unittests/results_dashboard_test.py

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Source file for results_dashboard testcases."""
+
+import json
+import os
+import shutil
+import tempfile
+import unittest
+import urllib
+import urllib2
+
+import test_env  # pylint: disable=W0403,W0611
+
+from slave import results_dashboard
+from slave import slave_utils
+from testing_support.super_mox import mox
+
+
+class IsEncodedJson(mox.Comparator):
+  def __init__(self, expected_json):
+    self._json = expected_json
+
+  def equals(self, rhs):
+    rhs_json = urllib.unquote_plus(rhs.data.replace("data=", ""))
+    return sorted(json.loads(self._json)) == sorted(json.loads(rhs_json))
+
+  def __repr__(self):
+    return "<Is Request JSON %s>" % self._json
+
+
+class ResultsDashboardTest(unittest.TestCase):
+  def setUp(self):
+    super(ResultsDashboardTest, self).setUp()
+    self.mox = mox.Mox()
+    self.build_dir = tempfile.mkdtemp()
+    os.makedirs(os.path.join(self.build_dir, results_dashboard.CACHE_DIR))
+    self.cache_filename = os.path.join(self.build_dir,
+                                       results_dashboard.CACHE_DIR,
+                                       results_dashboard.CACHE_FILENAME)
+
+  def tearDown(self):
+    self.mox.UnsetStubs()
+    shutil.rmtree(self.build_dir)
+
+  def _SendResults(self, send_results_args, expected_new_json, cached_json,
+                   errors, expected_cache):
+    cache_file = open(self.cache_filename, "wb")
+    cache_file.write("\n".join(cached_json))

[James Simonsen, 2013/03/01 21:21:49]

I don't think there's anything that tests that the script can read and send the
right things after it's written them to a file. So instead of seeding this file
from the test, why don't you call SendResults() twice in a single test, but have
the first one fail, so it's forced to reuse what it's written.

[sullivan, 2013/03/04 16:38:36]

Done.

+    cache_file.close()
+    self.mox.StubOutWithMock(slave_utils, "GetActiveMaster")
+    slave_utils.GetActiveMaster().AndReturn("ChromiumPerf")
+    self.mox.StubOutWithMock(urllib2, "urlopen")
+    for json_line, error in zip(cached_json + [expected_new_json], errors):
+      if error:
+        urllib2.urlopen(IsEncodedJson(json_line)).AndRaise(error)
+      else:
+        urllib2.urlopen(IsEncodedJson(json_line))
+    self.mox.ReplayAll()
+    send_results_args.append(self.build_dir)
+    results_dashboard.SendResults(*send_results_args)
+    self.mox.VerifyAll()
+    cache_file = open(self.cache_filename, "rb")
+    actual_cache = cache_file.read()
+    cache_file.close()
+    self.assertEqual(expected_cache, actual_cache)

[James Simonsen, 2013/03/01 21:21:49]

And then you wouldn't need to test this. Just make sure the second call to
urlopen() has what you want.

[sullivan, 2013/03/04 16:38:36]

Done.

+
+  def test_SingleLogLine(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://perf-dashboard.appspot.com"]
+    expected_new_json = json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])
+    cached_json = []
+    errors = [None]
+    expected_cache = ""
+    self._SendResults(
+        args, expected_new_json, cached_json, errors, expected_cache)
+
+  def test_MultipleLogLines(self):
+    args = [
+        "bar-summary.dat", [
+            '{"traces": {"baz": ["100.0", "5.0"]},'
+            ' "rev": "12345", "webkit_rev": "6789"}',
+            '{"traces": {"box": ["101.0", "4.0"]},'
+            ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://perf-dashboard.appspot.com"]
+    expected_new_json = json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}, {
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/box",
+        "revision": "12345",
+        "value": "101.0",
+        "error": "4.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}])
+    cached_json = []
+    errors = [None]
+    expected_cache = ""
+    self._SendResults(
+        args, expected_new_json, cached_json, errors, expected_cache)
+
+  def test_ModifiedTraceNames(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"bar": ["100.0", "5.0"], "bar_ref": ["99.0", "2.0"],'
+         ' "baz/y": ["101.0", "3.0"], "notchanged": ["102.0", "1.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://perf-dashboard.appspot.com"]
+    expected_new_json = json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }},{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/ref",
+        "revision": "12345",
+        "value": "99.0",
+        "error": "2.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}, {
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz_y",
+        "revision": "12345",
+        "value": "101.0",
+        "error": "3.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }},{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/notchanged",
+        "revision": "12345",
+        "value": "102.0",
+        "error": "1.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}])
+    cached_json = []
+    errors = [None]
+    expected_cache = ""
+    self._SendResults(
+        args, expected_new_json, cached_json, errors, expected_cache)
+
+  def test_ByUrlGraph(self):
+    args = [
+        "bar_by_url-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://perf-dashboard.appspot.com"]
+    expected_new_json = json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])
+    cached_json = []
+    errors = [None]
+    expected_cache = ""
+    self._SendResults(
+        args, expected_new_json, cached_json, errors, expected_cache)
+
+  def test_FailureRetried(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://perf-dashboard.appspot.com"]
+    expected_new_json = json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])
+    cached_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "dromaeo/dom",
+        "revision": "78899",
+        "value": "200.0",
+        "error": "5.0",
+    }]), json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "dromaeo/jslib",
+        "revision": "78899",
+        "value": "300.0",
+        "error": "5.0",
+    }])]
+    errors = [None, None, None]
+    expected_cache = ""
+    self._SendResults(
+        args, expected_new_json, cached_json, errors, expected_cache)
+
+  def test_FailureCached(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://perf-dashboard.appspot.com"]
+    expected_new_json = json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])
+    cached_json = []
+    errors = [urllib2.URLError("reason")]
+    expected_cache = expected_new_json
+    self._SendResults(
+        args, expected_new_json, cached_json, errors, expected_cache)
+
+  def test_MultipleFaliures(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://perf-dashboard.appspot.com"]
+    expected_new_json = json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])
+    cached_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "dromaeo/dom",
+        "revision": "78899",
+        "value": "200.0",
+        "error": "5.0",
+    }])]
+    errors = [urllib2.URLError("reason"), None]
+    expected_cache = cached_json[0]
+    self._SendResults(
+        args, expected_new_json, cached_json, errors, expected_cache)
+
+
+if __name__ == '__main__':
+  unittest.main()