Add src-side launcher for telemetry_unittests

testing/scripts/common.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import contextlib
+import json
+import os
+import subprocess
+import tempfile
+
+
+SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
+SRC_DIR = os.path.abspath(
+    os.path.join(SCRIPT_DIR, os.path.pardir, os.path.pardir))
+
+
+# run-webkit-tests returns the number of failures as the return
+# code, but caps the return code at 101 to avoid overflow or colliding
+# with reserved values from the shell.
+MAX_FAILURES_EXIT_STATUS = 101
+
+
+def run_script(argv, mode_run, mode_compile_targets):
+  def parse_json(path):
+    with open(path) as f:
+      return json.load(f)
+  parser = argparse.ArgumentParser()
+  # TODO(phajdan.jr): Make build-config-fs required after passing it in recipe.
+  parser.add_argument('--build-config-fs')
+  parser.add_argument('--paths', type=parse_json, default={})
+  parser.add_argument('--properties', type=parse_json, default={})
+
+  subparsers = parser.add_subparsers()
+
+  run_parser = subparsers.add_parser('run')
+  run_parser.add_argument('--output', required=True)
+  run_parser.add_argument('--filter-file', type=argparse.FileType('r'))
+  run_parser.set_defaults(func=mode_run)
+
+  run_parser = subparsers.add_parser('compile_targets')
+  run_parser.add_argument('--output', required=True)

[iannucci, 2014/10/22 07:23:15]

type=argparse.FileType('w') ?

[Paweł Hajdan Jr., 2014/10/22 09:43:27]

Done.

+  run_parser.set_defaults(func=mode_compile_targets)
+
+  args = parser.parse_args(argv)
+  return args.func(args)
+
+
+def run_command(argv):
+  print 'Running %r' % argv
+  rc = subprocess.call(argv)
+  print 'Command %r returned exit code %d' % (argv, rc)
+  return rc
+
+
+@contextlib.contextmanager
+def temporary_file():
+  fd, path = tempfile.mkstemp()
+  os.close(fd)
+  try:
+    yield path
+  finally:
+    os.remove(path)
+
+
+def parse_common_test_results(json_results):

[iannucci, 2014/10/22 07:23:14]

is this /really/ common? it seems like it's got a lot of webkit specific stuff

[Paweł Hajdan Jr., 2014/10/22 09:43:27]

As part of Ojan's CY work all test harnesses are standardizing on this format
(which indeed is originating from webkit). Please see PythonBasedTest in
steps.py, now several types of steps use this format, including python mojo
tests and telemetry.

+  def convert_trie_to_flat_paths(trie, prefix=None):

[iannucci, 2014/10/22 07:23:14]

lol, I think this function is in like 4 locations now

[Paweł Hajdan Jr., 2014/10/22 09:43:27]

Acknowledged.

+    # Also see webkitpy.layout_tests.layout_package.json_results_generator
+    result = {}
+    for name, data in trie.iteritems():
+      if prefix:
+        name = prefix + '/' + name
+      if len(data) and not 'actual' in data and not 'expected' in data:
+        result.update(convert_trie_to_flat_paths(data, name))
+      else:
+        result[name] = data
+    return result
+
+  results = {
+    'passes': {},
+    'unexpected_passes': {},
+    'failures': {},
+    'unexpected_failures': {},
+    'flakes': {},
+    'unexpected_flakes': {},
+  }
+
+  # TODO(dpranke): crbug.com/357866 - we should simplify the handling of
+  # both the return code and parsing the actual results, below.
+
+  passing_statuses = ('PASS', 'SLOW', 'NEEDSREBASELINE',
+                        'NEEDSMANUALREBASELINE')
+
+  for test, result in convert_trie_to_flat_paths(
+      json_results['tests']).iteritems():
+    key = 'unexpected_' if result.get('is_unexpected') else ''
+    data = result['actual']
+    actual_results = data.split()
+    last_result = actual_results[-1]
+    expected_results = result['expected'].split()
+
+    if (len(actual_results) > 1 and
+        (last_result in expected_results or last_result in passing_statuses)):
+      key += 'flakes'
+    elif last_result in passing_statuses:
+      key += 'passes'
+      # TODO(dpranke): crbug.com/357867 ...  Why are we assigning result
+      # instead of actual_result here. Do we even need these things to be
+      # hashes, or just lists?
+      data = result
+    else:
+      key += 'failures'
+    results[key][test] = data
+
+  return results