Iteratively increase sample size for good/bad classification.

scripts/slave/recipes/bisect.expected/basic.json

Index: scripts/slave/recipes/bisect.expected/basic.json
diff --git a/scripts/slave/recipes/bisect.expected/basic.json b/scripts/slave/recipes/bisect.expected/basic.json
index f9d3a2861e9310d135845c019468612e495fe901..0399ae7c06d7f899b79e5177a5290c1fd4eb4351 100644
--- a/scripts/slave/recipes/bisect.expected/basic.json
+++ b/scripts/slave/recipes/bisect.expected/basic.json
@@ -509,9 +509,8 @@
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "e28dc0d49c331def2a3bbf3ddd0096eb51551155-test",
@@ -552,9 +551,8 @@
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "a6298e4afedbf2cd461755ea6f45b0ad64222222-test",
@@ -804,9 +802,8 @@
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111-test",
@@ -873,6 +870,42 @@
     ]
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[70, 70, 80, 90, 90]",
+      "[14, 15, 16]",
+      "0.01"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111.Checking sample difference",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[19, 20, 21]",
+      "[14, 15, 16]",
+      "0.01"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111.Checking sample difference (2)",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144",
@@ -1000,9 +1033,8 @@
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "fc6dfc7ff5b1073408499478969261b826441144-test",
@@ -1069,6 +1101,42 @@
     ]
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[70, 70, 80, 90, 90]",
+      "[68, 70, 72]",
+      "0.01"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144.Checking sample difference",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[14, 15, 16]",
+      "[68, 70, 72]",
+      "0.01"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144.Checking sample difference (2)",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Check bisect finished on revision fc6dfc7ff5b1073408499478969261b826441144"