Sends test results to new perf dashboard

scripts/slave/unittests/results_dashboard_test.py

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Source file for results_dashboard testcases."""
+
+import json
+import os
+import shutil
+import tempfile
+import unittest
+import urllib
+import urllib2
+
+import test_env  # pylint: disable=W0403,W0611
+
+from slave import results_dashboard
+from slave import slave_utils
+from testing_support.super_mox import mox
+
+
+class IsEncodedJson(mox.Comparator):
+  def __init__(self, expected_json):
+    self._json = expected_json
+
+  def equals(self, rhs):
+    rhs_json = urllib.unquote_plus(rhs.data.replace("data=", ""))
+    return sorted(json.loads(self._json)) == sorted(json.loads(rhs_json))
+
+  def __repr__(self):
+    return "<Is Request JSON %s>" % self._json
+
+
+class ResultsDashboardTest(unittest.TestCase):
+  def setUp(self):
+    super(ResultsDashboardTest, self).setUp()
+    self.mox = mox.Mox()
+    self.build_dir = tempfile.mkdtemp()
+    os.makedirs(os.path.join(self.build_dir, results_dashboard.CACHE_DIR))
+    self.cache_filename = os.path.join(self.build_dir,
+                                       results_dashboard.CACHE_DIR,
+                                       results_dashboard.CACHE_FILENAME)
+
+  def tearDown(self):
+    self.mox.UnsetStubs()
+    shutil.rmtree(self.build_dir)
+
+  def _SendResults(self, send_results_args, expected_new_json, errors):
+    self.mox.UnsetStubs()  # Needed for multiple calls from same test.
+    self.mox.StubOutWithMock(slave_utils, "GetActiveMaster")
+    slave_utils.GetActiveMaster().AndReturn("ChromiumPerf")
+    self.mox.StubOutWithMock(urllib2, "urlopen")
+    for json_line, error in zip(expected_new_json, errors):
+      if error:
+        urllib2.urlopen(IsEncodedJson(json_line)).AndRaise(error)
+      else:
+        urllib2.urlopen(IsEncodedJson(json_line))
+    self.mox.ReplayAll()
+    send_results_args.append(self.build_dir)
+    results_dashboard.SendResults(*send_results_args)
+    self.mox.VerifyAll()
+
+  def test_SingleLogLine(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])]
+    errors = [None]
+    self._SendResults(args, expected_new_json, errors)
+
+  def test_MultipleLogLines(self):
+    args = [
+        "bar-summary.dat", [
+            '{"traces": {"baz": ["100.0", "5.0"]},'
+            ' "rev": "12345", "webkit_rev": "6789"}',
+            '{"traces": {"box": ["101.0", "4.0"]},'
+            ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}, {
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/box",
+        "revision": "12345",
+        "value": "101.0",
+        "error": "4.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}])]
+    errors = [None]
+    self._SendResults(args, expected_new_json, errors)
+
+  def test_ModifiedTraceNames(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"bar": ["100.0", "5.0"], "bar_ref": ["99.0", "2.0"],'
+         ' "baz/y": ["101.0", "3.0"], "notchanged": ["102.0", "1.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }},{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/ref",
+        "revision": "12345",
+        "value": "99.0",
+        "error": "2.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}, {
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz_y",
+        "revision": "12345",
+        "value": "101.0",
+        "error": "3.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }},{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/notchanged",
+        "revision": "12345",
+        "value": "102.0",
+        "error": "1.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+    }}])]
+    errors = [None]
+    self._SendResults(args, expected_new_json, errors)
+
+  def test_ByUrlGraph(self):
+    args = [
+        "bar_by_url-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])]
+    errors = [None]
+    self._SendResults(args, expected_new_json, errors)
+
+  def test_FailureRetried(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])]
+    errors = [urllib2.URLError("reason")]
+    self._SendResults(args, expected_new_json, errors)
+    args2 = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["101.0", "6.0"]},'
+         ' "rev": "12346", "webkit_rev": "6790"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json.append(json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12346",
+        "value": "101.0",
+        "error": "6.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6790",
+        }
+    }]))
+    errors = [None, None]
+    self._SendResults(args2, expected_new_json, errors)
+
+  def test_SuccessNotRetried(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])]
+    errors = [None]
+    self._SendResults(args, expected_new_json, errors)
+    args2 = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["101.0", "6.0"]},'
+         ' "rev": "12346", "webkit_rev": "6790"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json2 = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12346",
+        "value": "101.0",
+        "error": "6.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6790",
+        }
+    }])]
+    errors = [None]
+    self._SendResults(args2, expected_new_json2, errors)
+
+  def test_FailureCached(self):
+    args = [
+        "bar-summary.dat",
+        ['{"traces": {"baz": ["100.0", "5.0"]},'
+         ' "rev": "12345", "webkit_rev": "6789"}'],
+        "linux-release",
+        "foo",
+        "https://chrome-perf.googleplex.com"]
+    expected_new_json = [json.dumps([{
+        "master": "ChromiumPerf",
+        "bot": "linux-release",
+        "test": "foo/bar/baz",
+        "revision": "12345",
+        "value": "100.0",
+        "error": "5.0",
+        "supplemental_columns": {
+            "r_webkit_rev": "6789",
+        }}])]
+    errors = [urllib2.URLError("reason")]
+    self._SendResults(args, expected_new_json, errors)
+    cache_file = open(self.cache_filename, "rb")
+    actual_cache = cache_file.read()

[James Simonsen, 2013/03/04 19:03:24]

I was hoping you'd call SendResults again here, but this time let it succeed and
expect that you get the same JSON again. And/or a test that adds a new data
point and sends the combined list. That test would be more representative of
what we want to see.

[sullivan, 2013/03/04 20:00:57]

I do call SendResults() twice and check that it sends the combined result in
test_FailureRetried() above.

I wanted to check the cache here because it is important that we actually do
store the cache in an expected location in case there is some problem that
requires human intervention. For example, test results from dozens of runs pile
up because a new bot came online and we forgot to add the IP to the whitelist,
and now we have a ton of data and want to manually make sure it all gets sent.

+    cache_file.close()
+    self.assertEqual(expected_new_json[0], actual_cache)
+
+
+if __name__ == '__main__':
+  unittest.main()