Iteratively increase sample size for good/bad classification.

scripts/slave/recipe_modules/auto_bisect/bisector.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import re
 
 from . import bisect_results
 from . import depot_config
 from . import revision_state
@@ skipping 96 matching lines @@
       self.good_rev.good = True
       self.good_rev.read_deps(self.get_perf_tester_name())
       api.m.step.active_result.presentation.logs['Debug Good Revision DEPS'] = [
           '%s: %s' % (key, value) for key, value in
           self.good_rev.deps.iteritems()]
       self.good_rev.deps = {}
       self.lkgr = self.good_rev
     if init_revisions:
       self._expand_chromium_revision_range()
 
+  def significantly_different(self, list_a, list_b,
+                              significance_level=0.05): # pragma: no cover
+    step_result = self.api.m.python(
+        'Checking sample difference',
+        self.api.resource('significantly_different.py'),
+        [json.dumps(list_a), json.dumps(list_b), str(significance_level)],
+        stdout=self.api.m.json.output())
+    results = step_result.stdout
+    if results is None:
+      assert self.dummy_builds
+      return True
+    significantly_different = results['significantly_different']
+    step_result.presentation.logs[str(significantly_different)] = [
+        'See json.output for details']

[qyearsley, 2016/01/26 19:26:00]

How will this show up on the build page? Will there be a log link (along with
stdio and json.output) named either "True" or "False"?

[RobertoCN, 2016/02/01 17:29:50]

https://goo.gl/photos/E9AuQkm9C8Mpjyvw7

+    return significantly_different
+
   def config_step(self):
     """Yields a simple echo step that outputs the bisect config."""
     api = self.api
     # bisect_config may come as a FrozenDict (which is not serializable).
     bisect_config = dict(self.bisect_config)
 
     def fix_windows_backslashes(s):
       backslash_regex = re.compile(r'(?<!\\)\\(?!\\)')
       return backslash_regex.sub(r'\\', s)
 
@@ skipping 658 matching lines @@
   def surface_result(self, result_string):
     assert result_string in VALID_RESULT_CODES
     prefix = 'B4T_'  # To avoid collision. Stands for bisect (abbr. `a la i18n).
     result_code = prefix + result_string
     assert len(result_code) <= 20
     if result_code not in self.result_codes:
       self.result_codes.add(result_code)
       properties = self.api.m.step.active_result.presentation.properties
       properties['extra_result_code'] = sorted(self.result_codes)