Iteratively increase sample size for good/bad classification.

scripts/slave/recipes/bisect.expected/basic.json

 [
   {
     "cmd": [
       "python",
       "-u",
       "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
       "--master",
       "tryserver.chromium.perf",
       "--builder",
       "linux_perf_bisector",
@@ skipping 490 matching lines @@
   {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Gathering reference values.Triggering test job for 306475",
     "trigger_specs": [
       {
         "builder_name": "linux_perf_tester",
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "e28dc0d49c331def2a3bbf3ddd0096eb51551155-test",
           "parent_build_archive_url": "gs://chrome-perf/Linux Builder/full-build-linux_e28dc0d49c331def2a3bbf3ddd0096eb51551155.zip",
           "parent_got_revision": "e28dc0d49c331def2a3bbf3ddd0096eb51551155",
           "revision": "e28dc0d49c331def2a3bbf3ddd0096eb51551155"
         }
       }
     ],
     "~followup_annotations": [
@@ skipping 20 matching lines @@
   {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Gathering reference values.Triggering test job for src@a6298e4afedbf2cd461755ea6f45b0ad64222222",
     "trigger_specs": [
       {
         "builder_name": "linux_perf_tester",
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "a6298e4afedbf2cd461755ea6f45b0ad64222222-test",
           "parent_build_archive_url": "gs://chrome-perf/Linux Builder/full-build-linux_a6298e4afedbf2cd461755ea6f45b0ad64222222.zip",
           "parent_got_revision": "a6298e4afedbf2cd461755ea6f45b0ad64222222",
           "revision": "a6298e4afedbf2cd461755ea6f45b0ad64222222"
         }
       }
     ],
     "~followup_annotations": [
@@ skipping 229 matching lines @@
   {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Working on revision 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111.Triggering test job for 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111",
     "trigger_specs": [
       {
         "builder_name": "linux_perf_tester",
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111-test",
           "parent_build_archive_url": "gs://chrome-perf/Linux Builder/full-build-linux_00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111.zip",
           "parent_got_revision": "00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111",
           "revision": "00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111"
         }
       }
     ],
     "~followup_annotations": [
@@ skipping 46 matching lines @@
       "gs://chrome-perf/bisect-results/00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111-test.results"
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "Working on revision 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111.gsutil Get test results for build 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111",
     "stdout": "/path/to/tmp/",
     "~followup_annotations": [
       "@@@STEP_NEST_LEVEL@1@@@"
     ]
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[70, 70, 80, 90, 90]",
+      "[14, 15, 16]",
+      "0.05"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111.Checking sample difference",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[19, 20, 21]",
+      "[14, 15, 16]",
+      "0.05"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision 00316c9ddfb9d7b4e1ed2fff9fe6d964d2111111.Checking sample difference (2)",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@Partial Results@---partial bisect results start here---@@@",
       "@@@STEP_LOG_LINE@Partial Results@@@@",
       "@@@STEP_LOG_LINE@Partial Results@===== TESTED REVISIONS =====@@@",
       "@@@STEP_LOG_LINE@Partial Results@Depot    Revision Mean Value Std. Dev. Num Values Good?  @@@",
       "@@@STEP_LOG_LINE@Partial Results@chromium r306475  80.0       10.0      5          good   @@@",
       "@@@STEP_LOG_LINE@Partial Results@chromium r306477  15.0       1.0       3          bad    @@@",
@@ skipping 107 matching lines @@
   {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144.Triggering test job for fc6dfc7ff5b1073408499478969261b826441144",
     "trigger_specs": [
       {
         "builder_name": "linux_perf_tester",
         "properties": {
           "bisect_config": {
             "command": "tools/perf/run_benchmark -v --browser=release page_cycler.intl_ar_fa_he",
-            "max_time_minutes": "5",
             "metric": "warm_times/page_load_time",
-            "repeat_count": "2",
+            "repeat_count": 5,
             "test_type": "perf"
           },
           "job_name": "fc6dfc7ff5b1073408499478969261b826441144-test",
           "parent_build_archive_url": "gs://chrome-perf/Linux Builder/full-build-linux_fc6dfc7ff5b1073408499478969261b826441144.zip",
           "parent_got_revision": "fc6dfc7ff5b1073408499478969261b826441144",
           "revision": "fc6dfc7ff5b1073408499478969261b826441144"
         }
       }
     ],
     "~followup_annotations": [
@@ skipping 46 matching lines @@
       "gs://chrome-perf/bisect-results/fc6dfc7ff5b1073408499478969261b826441144-test.results"
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144.gsutil Get test results for build fc6dfc7ff5b1073408499478969261b826441144",
     "stdout": "/path/to/tmp/",
     "~followup_annotations": [
       "@@@STEP_NEST_LEVEL@1@@@"
     ]
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[70, 70, 80, 90, 90]",
+      "[68, 70, 72]",
+      "0.05"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144.Checking sample difference",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::auto_bisect]/resources/significantly_different.py",
+      "[14, 15, 16]",
+      "[68, 70, 72]",
+      "0.05"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "Working on revision fc6dfc7ff5b1073408499478969261b826441144.Checking sample difference (2)",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
     "cmd": [],
     "cwd": "[SLAVE_BUILD]",
     "name": "Check bisect finished on revision fc6dfc7ff5b1073408499478969261b826441144"
   },
   {
     "cmd": [
       "git",
       "show",
       "--name-only",
       "--pretty=format:",
@@ skipping 68 matching lines @@
       "@@@STEP_LOG_LINE@python.inline@shutil.copyfileobj(open(sys.argv[1]), sys.stdout)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
   {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
   }
 ]