Adding ability to bisect recipe to bisect into dependency repos.

scripts/slave/recipe_modules/auto_bisect/example.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+
+DEPS = [
+    'auto_bisect',
+    'path',
+    'properties',
+    'raw_io',
+    'step',
+]
+
+
+def GenSteps(api):
+  # Setting `api.path['checkout']` would ordinarily be done by
+  # `api.chromium_tests.sync_and_configure_build`
+  fake_checkout_path = api.path.mkdtemp('fake_checkout')
+  api.path['checkout'] = fake_checkout_path
+  bisector = api.auto_bisect.create_bisector(api.properties['bisect_config'])
+
+  # Request builds/tests for initial range and wait
+  bisector.good_rev.start_job()
+  bisector.bad_rev.start_job()
+  bisector.wait_for_all([bisector.good_rev, bisector.bad_rev])
+
+  assert bisector.check_improvement_direction()
+  assert bisector.check_regression_confidence()
+  revisions_to_check = bisector.get_revisions_to_eval(1)
+  assert len(revisions_to_check) == 1
+  revisions_to_check[0].start_job()
+  bisector.wait_for_any(revisions_to_check)
+  bisector.check_bisect_finished(revisions_to_check[0])
+
+  # Evaluate inserted DEPS-modified revisions
+  revisions_to_check = bisector.get_revisions_to_eval(2)
+  if revisions_to_check:
+    revisions_to_check[0].start_job()
+    revisions_to_check[0].read_deps()  # Only added for coverage.
+  else:
+    raise api.step.StepFailure('Expected revisions to check.')
+  # TODO(robertocn): Add examples for the following operations
+  # Abort unnecesary jobs # Print results
+
+
+def GenTests(api):

[qyearsley, 2015/03/10 23:20:49]

Optional: You could add notes about when GenSteps and GenTests are usually
called, for the benefit of people that are reading this without much knowledge
of any of the other recipe modules.

[RobertoCN, 2015/03/13 20:55:59]

Done.

+  basic_data = _get_basic_test_data()
+  yield _make_test(api, basic_data, 'basic')
+
+  reversed_basic_data = _get_reversed_basic_test_data()
+  yield _make_test(api, reversed_basic_data, 'reversed_basic')
+
+  bad_git_hash_data = _get_basic_test_data()
+  bad_git_hash_data[1]['interned_hashes'] = {'003': '12345', '002': 'Bad Hash'}
+  yield _make_test(api, bad_git_hash_data, 'failed_git_hash_object')
+
+  missing_dep_data = _get_basic_test_data()
+  tricked_DEPS_file = ("vars={'v8_' + 'revision': '001'};"
+                       "deps = {'src/v8': 'v8.git@' + Var('v8_revision'),"
+                       "'src/third_party/WebKit': 'webkit.git@010'}")
+  missing_dep_data[0]['DEPS'] = tricked_DEPS_file
+  yield _make_test(api, missing_dep_data, 'missing_vars_entry')
+
+  missing_dep_data = _get_basic_test_data()
+  tricked_DEPS_file = ("vars={'v8_revision': '001'};"
+                       "deps = {'src/v8': 'v8.XXX@' + Var('v8_revision'),"
+                       "'src/third_party/WebKit': 'webkit.git@010'}")
+  missing_dep_data[0]['DEPS'] = tricked_DEPS_file
+  yield _make_test(api, missing_dep_data, 'missing_deps_entry')
+
+  bad_deps_syntax_data = _get_basic_test_data()
+  bad_deps_syntax_data[1]['DEPS']='raise RuntimeError("")'
+  yield _make_test(api, bad_deps_syntax_data, 'bad_deps_syntax')
+
+
+def _get_basic_test_data():
+  return [
+      {
+          'hash': 'a6298e4afedbf2cd461755ea6f45b0ad64222222',
+          'commit_pos': '314015',
+          'test_results': {
+              'results':{
+                  'mean': 20,
+                  'std_err': 1,
+                  'values': [19, 20, 21],
+              }
+          },
+          "DEPS": ("vars={'v8_revision': '001'};"
+                   "deps = {'src/v8': 'v8.git@' + Var('v8_revision'),"
+                   "'src/third_party/WebKit': 'webkit.git@010'}"),
+          'git_diff': {
+              '002': 'Dummy .diff contents 001 - 002',
+              '003': 'Dummy .diff contents 001 - 003',
+          },
+      },
+      {
+          'hash': 'dcdcdc0ff1122212323134879ddceeb1240b0988',
+          'commit_pos': '314016',
+          'test_results': {
+              'results':{
+                  'mean': 15,
+                  'std_err': 1,
+                  'values': [14, 15, 16],
+              }
+          },
+          'DEPS_change': 'True',
+          "DEPS": ("vars={'v8_revision': '004'};"
+                   "deps = {'src/v8': 'v8.git@' + Var('v8_revision'),"
+                   "'src/third_party/WebKit': 'webkit.git@010'}"),
+          'DEPS_interval': {'v8': '004 003 002'.split()},
+      },
+      {
+          'hash': '00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111',
+          'commit_pos': '314017',
+          'test_results': {
+              'results':{
+                  'mean': 15,
+                  'std_err': 1,
+                  'values': [14, 15, 16],
+              }
+          }
+      },
+  ]
+
+
+def _get_reversed_basic_test_data():
+  return [
+      {
+          'hash': '00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111',
+          'commit_pos': '314015',
+          'test_results': {
+              'results':{
+                  'mean': 20,
+                  'std_err': 1,
+                  'values': [19, 20, 21],
+              }
+          }
+      },
+      {
+          'hash': 'a6298e4afedbf2cd461755ea6f45b0ad64222222',
+          'commit_pos': '314016',
+          'test_results': {
+              'results':{
+                  'mean': 20,
+                  'std_err': 1,
+                  'values': [19, 20, 21],
+              }
+          },
+          "DEPS": ("vars={'v8_revision': '001'};"
+                   "deps = {'src/v8': 'v8.git@' + Var('v8_revision'),"
+                   "'src/third_party/WebKit': 'webkit.git@010'}"),
+          'git_diff': {
+              '002': 'Dummy .diff contents 001 - 002',
+              '003': 'Dummy .diff contents 001 - 003',
+          },
+      },
+      {
+          'hash': 'dcdcdc0ff1122212323134879ddceeb1240b0988',
+          'commit_pos': '314017',
+          'test_results': {
+              'results':{
+                  'mean': 15,
+                  'std_err': 1,
+                  'values': [14, 15, 16],
+              }
+          },
+          'DEPS_change': 'True',
+          "DEPS": ("vars={'v8_revision': '004'};"
+                   "deps = {'src/v8': 'v8.git@' + Var('v8_revision'),"
+                   "'src/third_party/WebKit': 'webkit.git@010'}"),
+          'DEPS_interval': {'v8': '004 003 002'.split()},
+      },
+  ]
+
+
+def _make_test(api, test_data, test_name):
+  basic_test =  api.test(test_name)
+  for revision_data in test_data:
+    for step_data in _get_step_data_for_revision(api, revision_data):
+      basic_test += step_data
+  basic_test += api.properties(bisect_config=_get_default_config())
+  return basic_test
+
+
+def _get_default_config():
+  example_config = {
+      'test_type':'perf',
+      'command':
+          ('src/tools/perf/run_benchmark -v --browser=release smoothness.'
+           'tough_scrolling_cases'),
+      'good_revision': '314015',
+      'bad_revision': '314017',
+      'metric': 'mean_input_event_latency/mean_input_event_latency',
+      'repeat_count': '2',
+      'max_time_minutes': '5',
+      'truncate_percent': '0',
+      'bug_id': '',
+      'gs_bucket': 'chrome-perf',
+      'builder_host': 'master4.golo.chromium.org',
+      'builder_port': '8341',
+      'dummy_builds': 'True',
+  }
+  return example_config
+
+
+def _get_step_data_for_revision(api, revision_data, include_build_steps=True):
+  """Generator that produces step patches for fake results."""
+  commit_pos = revision_data['commit_pos']
+  commit_hash = revision_data['hash']
+  test_results = revision_data['test_results']

[qyearsley, 2015/03/10 23:20:49]

You don't have to do anything about this comment, but since I wrote it out I'll
keep it here:

One alternative to having lists of dicts that have particular field names is to
have named tuples. Named tuples are lightweight immutable objects with
particular attributes; my readability reviewer suggested I do this for the
"anomaly dicts" that were returned by FindAnomalies on the dashboard, and the
result was defining a class like this:

ChangePoint = collections.namedtuple(
    'ChangePoint', (
        # The x-value of the first point after a step.
        'x_value',
        # Median of the segments before and after the change point.
        'median_before', 'median_after',
        # Number of points before and after the change point.
        'size_before', 'size_after',
        # X-values of the first and last point in the series window used.
        'window_start', 'window_end',
        # Relative change from before to after.
        'relative_change',
        # Standard deviation of points before.
        'std_dev_before',
        # Results of the Welch's t-test for values before and after.
        't_statistic', 'degrees_of_freedom', 'p_value'
    ))

And then FindAnomalies returned objects of type ChangePoint. I think the main
advantage of doing this is that instead of saying that this function takes a
"revision data dict" with fields "commit_pos", "hash", etc., you could say, for
example, that it takes a list of RevisionInfo objects or something.

[RobertoCN, 2015/03/13 20:55:59]

Acknowledged.

+
+  step_name = 'resolving commit_pos ' + commit_pos
+  yield api.step_data(step_name, stdout=api.raw_io.output('hash:' +
+                                                          commit_hash))
+
+  step_name = 'resolving hash ' + commit_hash
+  commit_pos_str = 'refs/heads/master@{#%s}' % commit_pos
+  yield api.step_data(step_name, stdout=api.raw_io.output(commit_pos_str))
+
+  if include_build_steps:
+    step_name = 'gsutil Get test results for build ' + commit_hash
+    yield api.step_data(step_name, stdout=api.raw_io.output(json.dumps(
+        test_results)))
+
+    step_name = 'Get test status for build ' + commit_hash
+    yield api.step_data(step_name, stdout=api.raw_io.output('Complete'))
+
+    step_name = 'gsutil Get test status url for build ' + commit_hash
+    yield api.step_data(step_name, stdout=api.raw_io.output('dummy/url'))
+
+    if revision_data.get('DEPS', False):
+      step_name = 'git cat-file %s:DEPS' % commit_hash
+      yield api.step_data(step_name, stdout=api.raw_io.output(
+          revision_data['DEPS']))
+
+    if 'git_diff' in revision_data:
+      for deps_rev, diff_file in revision_data['git_diff'].iteritems():
+        step_name = 'Generating patch for %s:DEPS to %s'
+        step_name %= (commit_hash, deps_rev)
+        yield api.step_data(step_name, stdout=api.raw_io.output(diff_file))
+
+    if 'DEPS_change' in revision_data:
+      step_name = 'Checking DEPS for ' + commit_hash
+      yield api.step_data(step_name, stdout=api.raw_io.output('DEPS'))
+
+    if 'DEPS_interval' in revision_data:
+      for depot_name, interval in revision_data['DEPS_interval'].iteritems():
+        for item in interval[1:]:
+          step_name = 'Hashing modified DEPS file with revision ' + item
+          file_hash = 'f412e8458'
+          if 'interned_hashes' in revision_data:
+            file_hash = revision_data['interned_hashes'][item]
+          yield api.step_data(step_name, stdout=api.raw_io.output(file_hash))
+        step_name = 'Expanding revision range for revision %s on depot %s'
+        step_name %= (interval[0], depot_name)
+        stdout = api.raw_io.output('\n'.join(interval))
+        yield api.step_data(step_name, stdout=stdout)
+