Refactor stages 1, 2 and test_api overhaul.

scripts/slave/recipe_modules/perf_try/api.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """API for the perf try job recipe module.
 
 This API is meant to enable the perf try job recipe on any chromium-supported
 platform for any test that can be run via buildbot, perf or otherwise.
 """
 
@@ skipping 158 matching lines @@
       self.m.gclient.c.revisions['src'] = str(revision)
       update_step = self.m.bot_update.ensure_checkout(
           suffix=str(revision), patch=False, update_presentation=False)
       assert update_step.json.output['did_run']
       self.m.chromium.runhooks(name='runhooks on %s' % str(revision))
 
     return update_step
 
   def _run_test(self, cfg, **kwargs):
     """Runs test from config and return results."""
-    values, overall_output, retcodes = self.m.bisect_tester.run_test(
+    all_values = self.m.bisect_tester.run_test(
         cfg, **kwargs)
-    all_values = self.m.bisect_tester.digest_run_results(values, retcodes, cfg)
     overall_success = True
     if (not kwargs.get('allow_flakes', True) and
         cfg.get('test_type', 'perf') != 'return_code'):
-      overall_success = all(v == 0 for v in retcodes)
+      overall_success = all(v == 0 for v in all_values['retcodes'])
     return {
         'results': all_values,
         'ret_code': overall_success,
-        'output': ''.join(overall_output)
+        'output': ''.join(all_values['output'])
     }
 
   def _build_and_run_tests(self, cfg, update_step, bot_db, revision_hash,
                            **kwargs):
     """Compiles binaries and runs tests for a given a revision."""
     with_patch = kwargs.get('name') == 'With Patch'
     update_step = self._checkout_revision(update_step, bot_db, revision_hash)
     if update_step.presentation.properties:
       revision_hash = update_step.presentation.properties['got_revision']
     revision = build_state.BuildState(self, revision_hash, with_patch)
@@ skipping 75 matching lines @@
     if 'good_revision' not in config and 'bad_revision' not in config:
       return (None, None)
     return (self._get_hash(config.get('bad_revision')),
             self._get_hash(config.get('good_revision')))
 
   def _compare_and_present_results(
       self, cfg, results_without_patch, results_with_patch, labels):
     """Parses results and creates Results step."""
     output_with_patch = results_with_patch.get('output')
     output_without_patch = results_without_patch.get('output')
-    values_with_patch = results_with_patch.get('results').get('values')
-    values_without_patch = results_without_patch.get('results').get('values')
+    values_with_patch = self.parse_values_only(
+        results_with_patch.get('results'),
+        cfg.get('metric'),
+        _output_format(cfg.get('command')),
+        step_test_data=lambda: self.m.json.test_api.output_stream([1, 1, 1]))
+    values_without_patch = self.parse_values_only(
+        results_without_patch.get('results'),
+        cfg.get('metric'),
+        _output_format(cfg.get('command')),
+        step_test_data=lambda: self.m.json.test_api.output_stream([9, 9, 9]))
 
     cloud_links_without_patch = self.parse_cloud_links(output_without_patch)
     cloud_links_with_patch = self.parse_cloud_links(output_with_patch)
 
     results_link = (cloud_links_without_patch['html'][0]
                     if cloud_links_without_patch['html'] else '')
 
     if not values_with_patch or not values_without_patch:
       step_result = self.m.step('Results', [])
       step_result.presentation.step_text = (
@@ skipping 67 matching lines @@
         'profiler': profiler_pattern.findall(output),
     }
     return results
 
 
   def get_result(self, config, results_without_patch, results_with_patch,
                  labels):
     """Returns the results as a dict."""
     output_with_patch = results_with_patch.get('output')
     output_without_patch = results_without_patch.get('output')
-    values_with_patch = results_with_patch.get('results').get('values')
-    values_without_patch = results_without_patch.get('results').get('values')
+    values_with_patch = self.parse_values_only(
+        results_with_patch.get('results'),
+        config.get('metric'),
+        _output_format(config.get('command')),
+        step_test_data=lambda: self.m.json.test_api.output_stream([1, 1, 1]))
+    values_without_patch = self.parse_values_only(
+        results_without_patch.get('results'),
+        config.get('metric'),
+        _output_format(config.get('command')),
+        step_test_data=lambda: self.m.json.test_api.output_stream([9, 9, 9]))
 
     cloud_links_without_patch = self.parse_cloud_links(output_without_patch)
     cloud_links_with_patch = self.parse_cloud_links(output_with_patch)
 
     cloud_link = (cloud_links_without_patch['html'][0]
                   if cloud_links_without_patch['html'] else '')
 
     results = {
         'try_job_id': config.get('try_job_id'),
         'status': 'completed',  # TODO(chrisphan) Get partial results state.
@@ skipping 57 matching lines @@
 
   def _get_build_url(self):
     properties = self.m.properties
     bot_url = properties.get('buildbotURL',
                              'http://build.chromium.org/p/chromium/')
     builder_name = urllib.quote(properties.get('buildername', ''))
     builder_number = str(properties.get('buildnumber', ''))
     return '%sbuilders/%s/builds/%s' % (bot_url, builder_name, builder_number)
 
 
+  def parse_values_only(self, results, metric, output_format, **kwargs):
+    """Parse the values for a given metric for the given results.
+
+    This is meant to be used by tryjobs with a metric."""
+    if not metric:
+      return None
+
+    if output_format == 'buildbot':
+      files = results['stdout_paths']
+    elif output_format == 'chartjson':
+      files = results['chartjson_paths']
+    elif output_format == 'valueset':
+      files = results['valueset_paths']
+    else:  # pragma: no cover
+      raise self.m.step.StepFailure('Unsupported format: ' + output_format)
+
+    # Apply str to files to constrain cmdline args to ascii, as this used to
+    # break when unicode things were passed instead.
+    args = [','.join(map(str, files)), str(metric), '--' + output_format]
+    script = self.m.path['catapult'].join(
+        'tracing', 'bin', 'parse_metric_cmdline')
+    return self.m.python(
+        'Parse metric values',
+        script=script,
+        args=args,
+        stdout=self.m.json.output(),
+        **kwargs).stdout
+
+
 def _validate_perf_config(config_contents, required_parameters):
   """Validates the perf config file contents.
 
   This is used when we're doing a perf try job, the config file is called
   run-perf-test.cfg by default.
 
   The parameters checked are the required parameters; any additional optional
   parameters won't be checked and validation will still pass.
 
   Args:
@@ skipping 42 matching lines @@
 
 def _prepend_src_to_path_in_command(test_cfg):
   command_to_run = []
   for v in test_cfg.get('command').split():
     if v in ['./tools/perf/run_benchmark',
              'tools/perf/run_benchmark',
              'tools\\perf\\run_benchmark']:
       v = 'src/tools/perf/run_benchmark'
     command_to_run.append(v)
   test_cfg.update({'command': ' '.join(command_to_run)})
+
+def _output_format(command):
+  """Determine the output format for a given command."""
+  if 'chartjson' in command:
+    return 'chartjson'
+  elif 'valueset' in command:
+    return 'valueset'
+  return 'buildbot'