Add json test results format support for SwarmingIsolatedScriptTest

scripts/slave/recipe_modules/swarming/tests/results_merger_unittest.py

+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import unittest
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# For 'test_env'.
+sys.path.insert(
+    0, os.path.abspath(os.path.join(THIS_DIR, '..', '..',  '..', 'unittests')))

[Sergiy Byelozyorov, 2016/10/05 15:36:06]

Please move close to actual 'import test_env' line below or move import line
here.

[nednguyen, 2016/10/05 18:50:53]

Done.

+
+sys.path.insert(0, os.path.join(THIS_DIR, '..', '..'))

[Sergiy Byelozyorov, 2016/10/05 15:36:06]

What is this one for?

[nednguyen, 2016/10/05 18:50:53]

For importing resource_merger. PTAL again

+
+# Imported for side effects on sys.path.
+import test_env
+
+from swarming import results_merger
+
+
+GOOD_JSON_TEST_RESULT_0 = {
+  'tests': {
+    'car': {
+      'honda': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      },
+      'toyota': {
+        'expected': 'FAIL',
+        'actual': 'FAIL'
+      }
+    },
+    'computer': {
+      'dell': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    },
+  },
+  'interrupted': False,
+  'path_delimiter': '.',
+  'version': 3,
+  'seconds_since_epoch': 1406662289.76,
+  'num_failures_by_type': {
+     'FAIL': 0,
+     'PASS': 2
+  }
+}
+
+GOOD_JSON_TEST_RESULT_1 = {
+  'tests': {
+    'car': {
+      'tesla': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      },
+    },
+    'burger': {
+      'mcdonald': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    },
+  },
+  'interrupted': False,
+  'path_delimiter': '.',
+  'version': 3,
+  'seconds_since_epoch': 1406662283.11,
+  'num_failures_by_type': {
+     'FAIL': 0,
+     'PASS': 2
+  }
+}
+
+GOOD_JSON_TEST_RESULT_2 = {
+  'tests': {
+    'car': {
+      'mercedes': {
+        'expected': 'PASS',
+        'actual': 'FAIL'
+      },
+    },
+    'burger': {
+      'in n out': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    },
+  },
+  'interrupted': True,
+  'path_delimiter': '.',
+  'version': 3,
+  'seconds_since_epoch': 1406662200.01,
+  'num_failures_by_type': {
+     'FAIL': 1,
+     'PASS': 1
+  }
+}
+
+GOOD_JSON_TEST_RESULT_SLASH_DELIMITER_3 = {
+  'tests': {
+    'car': {
+      'mustang': {
+        'expected': 'PASS',
+        'actual': 'FAIL'
+      },
+    },
+    'burger': {
+      'white castle': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    },
+  },
+  'interrupted': True,
+  'path_delimiter': '/',
+  'version': 3,
+  'seconds_since_epoch': 1406662200.01,
+  'num_failures_by_type': {
+     'FAIL': 1,
+     'PASS': 1
+  }
+}
+
+GOOD_JSON_TEST_RESULT_MERGED = {
+  'tests': {
+    'car': {
+      'tesla': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      },
+      'mercedes': {
+        'expected': 'PASS',
+        'actual': 'FAIL'
+      },
+      'honda': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      },
+      'toyota': {
+        'expected': 'FAIL',
+        'actual': 'FAIL'
+      }
+    },
+    'computer': {
+      'dell': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    },
+    'burger': {
+      'mcdonald': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      },
+      'in n out': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    }
+  },
+  'interrupted': True,
+  'path_delimiter': '.',
+  'version': 3,
+  'seconds_since_epoch': 1406662200.01,
+  'num_failures_by_type': {
+    'FAIL': 1,
+    'PASS': 5
+  }
+}
+
+INVALID_JSON_TEST_RESULT_UNSUPPORTED_VERSION = {
+  'tests': {
+    'car': {
+      'tesla': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    },
+    'computer': {
+      'dell': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    },
+    'burger': {
+      'mcdonald': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      },
+      'in n out': {
+        'expected': 'PASS',
+        'actual': 'PASS'
+      }
+    }
+  },
+  'interrupted': True,
+  'path_delimiter': '.',
+  'version': 5,
+  'seconds_since_epoch': 1406662200.01,
+  'num_failures_by_type': {
+    'FAIL': 1,
+    'PASS': 5
+  }
+}
+
+
+
+class MergingTest(unittest.TestCase):  # pragma: no cover

[Sergiy Byelozyorov, 2016/10/05 15:36:06]

Why is this marked to be without coverage? Aren't we running these tests?

[nednguyen, 2016/10/05 18:50:53]

We aren't running these test in expectation framework, but are running them in
PRESUBMIT:

https://cs.chromium.org/chromium/build/PRESUBMIT.py?rcl=0&l=121

[Sergiy Byelozyorov, 2016/10/09 14:43:10]

This is sad that we have to misuse pragma-no-cover here, but for lack of a
better alternative, I guess this will do. Can you please add a comment
explaining that?

[nednguyen, 2016/10/10 13:24:52]

Done.

+  def test_merge_json_test_results_format_ok(self):
+    self.maxDiff = None  # Show full diff if assertion fail
+    self.assertEquals(results_merger.merge_test_results(
+        [GOOD_JSON_TEST_RESULT_0,
+         GOOD_JSON_TEST_RESULT_1,
+         GOOD_JSON_TEST_RESULT_2]),
+        GOOD_JSON_TEST_RESULT_MERGED)
+
+  def test_merge_unsupported_json_test_results_format(self):
+    with self.assertRaises(Exception):
+      results_merger.merge_test_results(
+        [GOOD_JSON_TEST_RESULT_0, INVALID_JSON_TEST_RESULT_0])
+
+  def test_merge_incompatible_json_test_results_format(self):
+    with self.assertRaises(Exception):
+      results_merger.merge_test_results(
+        [GOOD_JSON_TEST_RESULT_0, GOOD_JSON_TEST_RESULT_SLASH_DELIMITER_3])
+
+if __name__ == '__main__':
+  unittest.main()  # pragma: no cover