Adding ability to bisect recipe to bisect into dependency repos.

scripts/slave/recipe_modules/auto_bisect/example.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+
+DEPS = [
+    'auto_bisect',
+    'path',
+    'properties',
+    'raw_io',
+]
+
+
+def GenSteps(api):

[qyearsley, 2015/02/27 23:12:02]

General question: When in example.GenSteps called?

[RobertoCN, 2015/03/03 21:45:30]

Example is just a recipe like any other. GenSteps is called when running the
recipe (in the case of example it is run when called from
recipe_simulation_test)

+  # Setting `api.path['checkout']` would ordinarily be done by
+  # `api.chromium_tests.sync_and_configure_build`
+  fake_checkout_path = api.path.mkdtemp('fake_checkout')
+  api.path['checkout'] = fake_checkout_path
+  # Create a bisector
+  bisector = api.auto_bisect.create_bisector(api.properties['bisect_config'])
+  # Request builds
+  bisector.good_rev.start_job()
+  bisector.bad_rev.start_job()
+  # Wait for builds and tests to finish
+  bisector.wait_for_all([bisector.good_rev, bisector.bad_rev])
+
+  # TODO(robertocn): Add examples for the following operations
+  # Check improvement direction
+  assert bisector.check_improvement_direction()
+  # Check regression confidence

[qyearsley, 2015/02/27 23:12:02]

Some of these comments are probably unnecessary.

[RobertoCN, 2015/03/03 21:45:30]

Done.

+  assert bisector.check_regression_confidence()
+  # Get revisions to check
+  revisions_to_check = bisector.get_revisions_to_eval(1)
+  assert len(revisions_to_check) == 1
+  revisions_to_check[0].start_job()
+  bisector.wait_for_any(revisions_to_check)
+  # Check if bisect is finished
+  bisector.check_bisect_finished(revisions_to_check[0])
+  # Abort unnecesary jobs
+  # Print results
+
+
+def GenTests(api):
+  yield _basic_test(api)
+
+
+def _basic_test(api):
+  test_data = [
+      {
+          'hash': 'a6298e4afedbf2cd461755ea6f45b0ad64222222',
+          'commit_pos': '314015',
+          'test_results': {
+              'results':{
+                  'mean': 20,
+                  'std_err': 1,
+                  'values': [19, 20, 21],
+              }
+          },
+          "DEPS": ("vars={'v8_revision': '001'};"
+                   "deps = {'src/v8': 'v8.git@' + Var('v8_revision'),"
+                   "'src/third_party/WebKit': 'webkit.git@010'}"),
+          'git_diff': {
+              '002': 'Dummy .diff contents 001 - 002',
+              '003': 'Dummy .diff contents 001 - 003',
+          },
+      },
+      {
+          'hash': 'dcdcdc0ff1122212323134879ddceeb1240b0988',
+          'commit_pos': '314016',
+          'test_results': {
+              'results':{
+                  'mean': 15,
+                  'std_err': 1,
+                  'values': [14, 15, 16],
+              }
+          },
+          'DEPS_change': 'True',
+          "DEPS": ("vars={'v8_revision': '004'};"
+                   "deps = {'src/v8': 'v8.git@' + Var('v8_revision'),"
+                   "'src/third_party/WebKit': 'webkit.git@010'}"),
+          'DEPS_interval': {'v8': '004 003 002'.split()},
+      },
+      {
+          'hash': '00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111',
+          'commit_pos': '314017',
+          'test_results': {
+              'results':{
+                  'mean': 15,
+                  'std_err': 1,
+                  'values': [14, 15, 16],
+              }
+          }
+      },
+  ]
+  basic_test =  api.test('basic')
+  for revision_data in test_data:
+    for step_data in _get_step_data_for_revision(api, revision_data):
+      basic_test += step_data

[iannucci, 2015/03/18 23:43:28]

basic_test += sum(_get_step_data_for_revision(api, revision_data))

probably works too, I think.

Not sure that _get_step_data_for_revision needs to be a generator.

[robertocn1, 2015/03/19 21:48:23]

Done.

+  basic_test += api.properties(bisect_config=_get_default_config())
+  return basic_test
+
+
+def _get_default_config():
+  example_config = {
+      'test_type':'perf',
+      'command':
+          ('src/tools/perf/run_benchmark -v --browser=release smoothness.'
+           'tough_scrolling_cases'),
+      'good_revision': '314015',
+      'bad_revision': '314017',
+      'metric': 'mean_input_event_latency/mean_input_event_latency',
+      'repeat_count': '2',
+      'max_time_minutes': '5',
+      'truncate_percent': '0',
+      'bug_id': '',
+      'gs_bucket': 'chrome-perf',
+      'builder_host': 'master4.golo.chromium.org',
+      'builder_port': '8341',
+      'dummy_builds': 'True',
+  }
+  return example_config
+
+
+def _get_step_data_for_revision(api, revision_data, include_build_steps=True):
+  """Generator that produces step patches for fake results."""
+  commit_pos = revision_data['commit_pos']
+  commit_hash = revision_data['hash']
+  test_results = revision_data['test_results']
+
+  step_name ='resolving commit_pos ' + commit_pos
+  yield api.step_data(step_name, stdout=api.raw_io.output('hash:' +
+                                                            commit_hash))

[qyearsley, 2015/02/27 23:12:02]

Style nit: indentation seems off.

[RobertoCN, 2015/03/03 21:45:30]

Done.

+
+  step_name ='resolving hash ' + commit_hash

[qyearsley, 2015/02/27 23:12:02]

Style nit: space after "=".

[RobertoCN, 2015/03/03 21:45:30]

Done.

+  commit_pos_str = 'refs/heads/master@{#%s}' % commit_pos
+  yield api.step_data(step_name, stdout=api.raw_io.output(commit_pos_str))
+
+  if include_build_steps:
+    step_name ='gsutil Get test results for build ' + commit_hash

[qyearsley, 2015/02/27 23:12:03]

Style nit: space after "=".

[RobertoCN, 2015/03/03 21:45:30]

Done.

+    yield api.step_data(step_name, stdout=api.raw_io.output(json.dumps(
+        test_results)))
+
+    step_name = 'Get test status for build ' + commit_hash
+    yield api.step_data(step_name, stdout=api.raw_io.output('Complete'))
+
+    step_name = 'gsutil Get test status url for build ' + commit_hash
+    yield api.step_data(step_name, stdout=api.raw_io.output('dummy/url'))
+
+    if revision_data.get('DEPS', False):

[qyearsley, 2015/02/27 23:12:02]

Equivalent:

if revision_data.get('DEPS'):
  ...

if 'DEPS' in revision_data:
  ...

[RobertoCN, 2015/03/03 21:45:30]

Done.

+      step_name = 'git cat-file %s:DEPS' % commit_hash
+      yield api.step_data(step_name, stdout=api.raw_io.output(
+          revision_data['DEPS']))
+
+    if revision_data.get('git_diff', False):
+      for deps_rev, diff_file in revision_data['git_diff'].iteritems():
+        step_name = 'Generating patch for %s:DEPS to %s'
+        step_name %= (commit_hash, deps_rev)
+        yield api.step_data(step_name, stdout=api.raw_io.output(diff_file))
+
+    if revision_data.get('DEPS_change', False):
+      step_name = 'Checking DEPS for ' + commit_hash
+      yield api.step_data(step_name, stdout=api.raw_io.output('DEPS'))
+
+    if revision_data.get('DEPS_interval', False):
+      for depot_name, interval in revision_data['DEPS_interval'].iteritems():
+        for item in interval[1:]:
+          step_name = 'Hashing modified DEPS file with revision ' + item
+          yield api.step_data(step_name, stdout=api.raw_io.output('f412e8458'))
+        step_name = 'Expanding revision range for revision %s on depot %s'
+        step_name %= (interval[0], depot_name)
+        stdout=api.raw_io.output('\n'.join(interval))

[qyearsley, 2015/02/27 23:12:03]

Style nit: spaces around "=".

[RobertoCN, 2015/03/03 21:45:30]

Done.

+        yield api.step_data(step_name, stdout=stdout)
+