| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 from common import constants | 5 from common import constants |
| 6 from common.pipeline_wrapper import pipeline_handlers | 6 from common.pipeline_wrapper import pipeline_handlers |
| 7 from model import analysis_status | 7 from model import analysis_status |
| 8 from model.flake.master_flake_analysis import MasterFlakeAnalysis | 8 from model.flake.master_flake_analysis import MasterFlakeAnalysis |
| 9 from model.flake.flake_swarming_task import FlakeSwarmingTask |
| 9 from waterfall.flake import recursive_flake_pipeline | 10 from waterfall.flake import recursive_flake_pipeline |
| 11 from waterfall.flake.recursive_flake_pipeline import get_next_run |
| 10 from waterfall.flake.recursive_flake_pipeline import NextBuildNumberPipeline | 12 from waterfall.flake.recursive_flake_pipeline import NextBuildNumberPipeline |
| 11 from waterfall.flake.recursive_flake_pipeline import RecursiveFlakePipeline | 13 from waterfall.flake.recursive_flake_pipeline import RecursiveFlakePipeline |
| 14 from waterfall.flake.recursive_flake_pipeline import sequential_next_run |
| 12 from waterfall.test import wf_testcase | 15 from waterfall.test import wf_testcase |
| 13 | 16 |
| 14 | 17 |
| 15 class RecursiveFlakePipelineTest(wf_testcase.WaterfallTestCase): | 18 class RecursiveFlakePipelineTest(wf_testcase.WaterfallTestCase): |
| 16 app_module = pipeline_handlers._APP | 19 app_module = pipeline_handlers._APP |
| 17 | 20 |
| 18 def _CreateAndSaveMasterFlakeAnalysis( | 21 def _CreateAndSaveMasterFlakeAnalysis( |
| 19 self, master_name, builder_name, build_number, | 22 self, master_name, builder_name, build_number, |
| 20 step_name, test_name, status): | 23 step_name, test_name, status): |
| 21 analysis = MasterFlakeAnalysis.Create( | 24 analysis = MasterFlakeAnalysis.Create( |
| 22 master_name, builder_name, build_number, step_name, test_name) | 25 master_name, builder_name, build_number, step_name, test_name) |
| 23 analysis.status = status | 26 analysis.status = status |
| 24 analysis.put() | 27 analysis.put() |
| 25 | 28 |
| 29 def _CreateAndSaveFlakeSwarmingTask( |
| 30 self, master_name, builder_name, build_number, |
| 31 step_name, test_name, status): |
| 32 flake_swarming_task = FlakeSwarmingTask.Create( |
| 33 master_name, builder_name, build_number, step_name, test_name) |
| 34 flake_swarming_task.status = status |
| 35 flake_swarming_task.put() |
| 36 |
| 26 def testRecursiveFlakePipeline(self): | 37 def testRecursiveFlakePipeline(self): |
| 27 master_name = 'm' | 38 master_name = 'm' |
| 28 builder_name = 'b' | 39 builder_name = 'b' |
| 29 master_build_number = 124 | 40 master_build_number = 100 |
| 30 build_number = 124 | 41 build_number = 100 |
| 31 run_build_number = 124 | 42 run_build_number = 100 |
| 32 step_name = 's' | 43 step_name = 's' |
| 33 test_name = 't' | 44 test_name = 't' |
| 34 test_result_future = 'test_result_future' | 45 test_result_future = 'test_result_future' |
| 35 queue_name = constants.DEFAULT_QUEUE | 46 queue_name = constants.DEFAULT_QUEUE |
| 36 task_id = 'task_id' | 47 task_id = 'task_id' |
| 37 | 48 |
| 49 flakiness_algorithm_results_dict = { |
| 50 'flakes_in_a_row': 0, |
| 51 'stable_in_a_row': 0, |
| 52 'stabled_out': False, |
| 53 'flaked_out': False, |
| 54 'last_build_number': 0, |
| 55 'lower_boundary': None, |
| 56 'upper_boundary': None, |
| 57 'lower_boundary_result': None, |
| 58 'sequential_run_index': 0 |
| 59 |
| 60 } |
| 61 |
| 38 self.MockPipeline( | 62 self.MockPipeline( |
| 39 recursive_flake_pipeline.TriggerFlakeSwarmingTaskPipeline, | 63 recursive_flake_pipeline.TriggerFlakeSwarmingTaskPipeline, |
| 40 'task_id', | 64 'task_id', |
| 41 expected_args=[master_name, builder_name, | 65 expected_args=[master_name, builder_name, |
| 42 run_build_number, step_name, [test_name]], | 66 run_build_number, step_name, [test_name]], |
| 43 expected_kwargs={}) | 67 expected_kwargs={}) |
| 68 |
| 44 self.MockPipeline( | 69 self.MockPipeline( |
| 45 recursive_flake_pipeline.ProcessFlakeSwarmingTaskResultPipeline, | 70 recursive_flake_pipeline.ProcessFlakeSwarmingTaskResultPipeline, |
| 46 'test_result_future', | 71 'test_result_future', |
| 47 expected_args=[master_name, builder_name, | 72 expected_args=[master_name, builder_name, |
| 48 run_build_number, step_name, task_id, | 73 run_build_number, step_name, task_id, |
| 49 master_build_number, test_name], | 74 master_build_number, test_name], |
| 50 expected_kwargs={}) | 75 expected_kwargs={}) |
| 76 |
| 51 self.MockPipeline( | 77 self.MockPipeline( |
| 52 recursive_flake_pipeline.NextBuildNumberPipeline, | 78 recursive_flake_pipeline.NextBuildNumberPipeline, |
| 53 '', | 79 '', |
| 54 expected_args=[master_name, builder_name, master_build_number, | 80 expected_args=[master_name, builder_name, master_build_number, |
| 55 step_name, test_name, test_result_future, | 81 build_number, step_name, test_name, |
| 56 queue_name], | 82 test_result_future, queue_name, |
| 83 flakiness_algorithm_results_dict], |
| 57 expected_kwargs={}) | 84 expected_kwargs={}) |
| 58 | 85 |
| 59 rfp = RecursiveFlakePipeline(master_name, builder_name, build_number, | 86 rfp = RecursiveFlakePipeline(master_name, builder_name, build_number, |
| 60 step_name, test_name, master_build_number, | 87 step_name, test_name, master_build_number, |
| 88 flakiness_algorithm_results_dict= |
| 89 flakiness_algorithm_results_dict, |
| 61 queue_name=queue_name) | 90 queue_name=queue_name) |
| 91 |
| 62 rfp.start(queue_name=queue_name) | 92 rfp.start(queue_name=queue_name) |
| 63 self.execute_queued_tasks() | 93 self.execute_queued_tasks() |
| 64 | 94 |
| 65 def testNextBuildPipelineForNewRecursion(self): | 95 def testNextBuildPipelineForNewRecursionFirstFlake(self): |
| 66 master_name = 'm' | 96 master_name = 'm' |
| 67 builder_name = 'b' | 97 builder_name = 'b' |
| 68 master_build_number = 124 | 98 master_build_number = 100 |
| 69 build_number = 124 | 99 build_number = 100 |
| 70 step_name = 's' | 100 step_name = 's' |
| 71 test_name = 't' | 101 test_name = 't' |
| 72 test_result_future = 'trf' | 102 test_result_future = 'trf' |
| 73 queue_name = constants.DEFAULT_QUEUE | 103 queue_name = constants.DEFAULT_QUEUE |
| 74 | 104 flakiness_algorithm_results_dict = { |
| 75 self._CreateAndSaveMasterFlakeAnalysis( | 105 'flakes_in_a_row': 0, |
| 76 master_name, builder_name, build_number, step_name, | 106 'stable_in_a_row': 0, |
| 77 test_name, status=analysis_status.PENDING | 107 'stabled_out': False, |
| 78 ) | 108 'flaked_out': False, |
| 79 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, | 109 'last_build_number': 0, |
| 80 build_number, step_name, test_name) | 110 'lower_boundary': None, |
| 81 analysis.build_numbers.append(124) | 111 'upper_boundary': None, |
| 112 'lower_boundary_result': None, |
| 113 'sequential_run_index': 0 |
| 114 |
| 115 } |
| 116 self._CreateAndSaveMasterFlakeAnalysis( |
| 117 master_name, builder_name, build_number, step_name, |
| 118 test_name, status=analysis_status.PENDING |
| 119 ) |
| 120 self._CreateAndSaveFlakeSwarmingTask( |
| 121 master_name, builder_name, build_number, step_name, |
| 122 test_name, status=analysis_status.COMPLETED |
| 123 ) |
| 124 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 125 build_number, step_name, test_name) |
| 126 analysis.success_rates.append(.08) |
| 127 analysis.build_numbers.append(100) |
| 128 analysis.put() |
| 129 |
| 130 NextBuildNumberPipeline.run( |
| 131 NextBuildNumberPipeline(), master_name, builder_name, |
| 132 master_build_number, build_number, step_name, test_name, |
| 133 test_result_future, queue_name, flakiness_algorithm_results_dict) |
| 134 self.assertEquals(flakiness_algorithm_results_dict['flakes_in_a_row'], 1) |
| 135 |
| 136 def testNextBuildPipelineForNewRecursionFirstStable(self): |
| 137 master_name = 'm' |
| 138 builder_name = 'b' |
| 139 master_build_number = 100 |
| 140 build_number = 100 |
| 141 step_name = 's' |
| 142 test_name = 't' |
| 143 test_result_future = 'trf' |
| 144 queue_name = constants.DEFAULT_QUEUE |
| 145 flakiness_algorithm_results_dict = { |
| 146 'flakes_in_a_row': 0, |
| 147 'stable_in_a_row': 0, |
| 148 'stabled_out': False, |
| 149 'flaked_out': False, |
| 150 'last_build_number': 0, |
| 151 'lower_boundary': None, |
| 152 'upper_boundary': None, |
| 153 'lower_boundary_result': None, |
| 154 'sequential_run_index': 0 |
| 155 |
| 156 } |
| 157 self._CreateAndSaveMasterFlakeAnalysis( |
| 158 master_name, builder_name, build_number, step_name, |
| 159 test_name, status=analysis_status.PENDING |
| 160 ) |
| 161 self._CreateAndSaveFlakeSwarmingTask( |
| 162 master_name, builder_name, build_number, step_name, |
| 163 test_name, status=analysis_status.COMPLETED |
| 164 ) |
| 165 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 166 build_number, step_name, test_name) |
| 167 analysis.success_rates.append(0) |
| 168 analysis.build_numbers.append(100) |
| 169 analysis.put() |
| 170 |
| 171 NextBuildNumberPipeline.run( |
| 172 NextBuildNumberPipeline(), master_name, builder_name, |
| 173 master_build_number, build_number, step_name, |
| 174 test_name, test_result_future, queue_name, |
| 175 flakiness_algorithm_results_dict) |
| 176 self.assertEquals(flakiness_algorithm_results_dict['stable_in_a_row'], 1) |
| 177 |
| 178 def testNextBuildPipelineForNewRecursionFlakeInARow(self): |
| 179 master_name = 'm' |
| 180 builder_name = 'b' |
| 181 master_build_number = 100 |
| 182 build_number = 100 |
| 183 step_name = 's' |
| 184 test_name = 't' |
| 185 test_result_future = 'trf' |
| 186 queue_name = constants.DEFAULT_QUEUE |
| 187 flakiness_algorithm_results_dict = { |
| 188 'flakes_in_a_row': 0, |
| 189 'stable_in_a_row': 4, |
| 190 'stabled_out': False, |
| 191 'flaked_out': False, |
| 192 'last_build_number': 0, |
| 193 'lower_boundary': None, |
| 194 'upper_boundary': None, |
| 195 'lower_boundary_result': None, |
| 196 'sequential_run_index': 0 |
| 197 |
| 198 } |
| 199 self._CreateAndSaveMasterFlakeAnalysis( |
| 200 master_name, builder_name, build_number, step_name, |
| 201 test_name, status=analysis_status.PENDING |
| 202 ) |
| 203 self._CreateAndSaveFlakeSwarmingTask( |
| 204 master_name, builder_name, build_number, step_name, |
| 205 test_name, status=analysis_status.COMPLETED |
| 206 ) |
| 207 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 208 build_number, step_name, test_name) |
| 209 analysis.success_rates.append(0) |
| 210 analysis.build_numbers.append(100) |
| 211 analysis.put() |
| 212 |
| 213 NextBuildNumberPipeline.run( |
| 214 NextBuildNumberPipeline(), master_name, builder_name, |
| 215 master_build_number, build_number, step_name, |
| 216 test_name, test_result_future, queue_name, |
| 217 flakiness_algorithm_results_dict) |
| 218 self.assertEquals(flakiness_algorithm_results_dict['stabled_out'], True) |
| 219 |
| 220 def testNextBuildPipelineForNewRecursionStableInARow(self): |
| 221 master_name = 'm' |
| 222 builder_name = 'b' |
| 223 master_build_number = 100 |
| 224 build_number = 100 |
| 225 step_name = 's' |
| 226 test_name = 't' |
| 227 test_result_future = 'trf' |
| 228 queue_name = constants.DEFAULT_QUEUE |
| 229 flakiness_algorithm_results_dict = { |
| 230 'flakes_in_a_row': 4, |
| 231 'stable_in_a_row': 0, |
| 232 'stabled_out': False, |
| 233 'flaked_out': False, |
| 234 'last_build_number': 0, |
| 235 'lower_boundary': None, |
| 236 'upper_boundary': None, |
| 237 'lower_boundary_result': None, |
| 238 'sequential_run_index': 0 |
| 239 |
| 240 } |
| 241 self._CreateAndSaveMasterFlakeAnalysis( |
| 242 master_name, builder_name, build_number, step_name, |
| 243 test_name, status=analysis_status.PENDING |
| 244 ) |
| 245 self._CreateAndSaveFlakeSwarmingTask( |
| 246 master_name, builder_name, build_number, step_name, |
| 247 test_name, status=analysis_status.COMPLETED |
| 248 ) |
| 249 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 250 build_number, step_name, test_name) |
| 251 analysis.success_rates.append(.50) |
| 252 analysis.build_numbers.append(100) |
| 253 analysis.put() |
| 254 |
| 255 NextBuildNumberPipeline.run( |
| 256 NextBuildNumberPipeline(), master_name, builder_name, |
| 257 master_build_number, build_number, step_name, |
| 258 test_name, test_result_future, queue_name, |
| 259 flakiness_algorithm_results_dict) |
| 260 self.assertEquals(flakiness_algorithm_results_dict['flaked_out'], True) |
| 261 |
| 262 |
| 263 def testNextBuildPipelineForNewRecursionLessThanLastBuildNumber(self): |
| 264 master_name = 'm' |
| 265 builder_name = 'b' |
| 266 master_build_number = 100 |
| 267 build_number = 100 |
| 268 step_name = 's' |
| 269 test_name = 't' |
| 270 test_result_future = 'trf' |
| 271 queue_name = constants.DEFAULT_QUEUE |
| 272 flakiness_algorithm_results_dict = { |
| 273 'flakes_in_a_row': 0, |
| 274 'stable_in_a_row': 0, |
| 275 'stabled_out': False, |
| 276 'flaked_out': False, |
| 277 'last_build_number': 200, |
| 278 'lower_boundary': None, |
| 279 'upper_boundary': None, |
| 280 'lower_boundary_result': None, |
| 281 'sequential_run_index': 0 |
| 282 } |
| 283 self._CreateAndSaveMasterFlakeAnalysis( |
| 284 master_name, builder_name, build_number, step_name, |
| 285 test_name, status=analysis_status.PENDING |
| 286 ) |
| 287 self._CreateAndSaveFlakeSwarmingTask( |
| 288 master_name, builder_name, build_number, step_name, |
| 289 test_name, status=analysis_status.COMPLETED |
| 290 ) |
| 291 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 292 build_number, step_name, test_name) |
| 293 analysis.success_rates.append(.50) |
| 294 analysis.build_numbers.append(100) |
| 82 analysis.put() | 295 analysis.put() |
| 83 | 296 |
| 84 queue_name = {'x': False} | 297 queue_name = {'x': False} |
| 85 # Unused argument (class method calls in python) - pylint: disable=W0613 | 298 def my_mocked_run(arg1, queue_name): # pylint: disable=unused-argument |
| 86 def my_mocked_run(arg1, queue_name): | 299 queue_name['x'] = True # pragma: no cover |
| 87 queue_name['x'] = True | |
| 88 | 300 |
| 89 self.mock( | 301 self.mock( |
| 90 recursive_flake_pipeline.RecursiveFlakePipeline, 'start', my_mocked_run) | 302 recursive_flake_pipeline.RecursiveFlakePipeline, 'start', my_mocked_run) |
| 91 NextBuildNumberPipeline.run( | 303 NextBuildNumberPipeline.run( |
| 92 NextBuildNumberPipeline(), master_name, builder_name, | 304 NextBuildNumberPipeline(), master_name, builder_name, |
| 93 master_build_number, step_name, test_name, test_result_future, | 305 master_build_number, build_number, step_name, test_name, |
| 94 queue_name) | 306 test_result_future, queue_name, flakiness_algorithm_results_dict) |
| 95 self.assertTrue(queue_name['x']) | 307 self.assertFalse(queue_name['x']) |
| 96 | 308 |
| 97 def testNextBuildPipelineForNewRecursionWhenDone(self): | 309 def testNextBuildPipelineForFailedSwarmingTask(self): |
| 98 master_name = 'm' | 310 master_name = 'm' |
| 99 builder_name = 'b' | 311 builder_name = 'b' |
| 100 master_build_number = 124 | 312 master_build_number = 100 |
| 101 build_number = 124 | 313 build_number = 100 |
| 102 step_name = 's' | 314 step_name = 's' |
| 103 test_name = 't' | 315 test_name = 't' |
| 104 test_result_future = 'trf' | 316 test_result_future = 'trf' |
| 105 queue_name = constants.DEFAULT_QUEUE | 317 queue_name = constants.DEFAULT_QUEUE |
| 106 | 318 flakiness_algorithm_results_dict = { |
| 107 self._CreateAndSaveMasterFlakeAnalysis( | 319 'flakes_in_a_row': 0, |
| 108 master_name, builder_name, build_number, step_name, | 320 'stable_in_a_row': 0, |
| 109 test_name, status=analysis_status.PENDING | 321 'stabled_out': False, |
| 110 ) | 322 'flaked_out': False, |
| 111 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, | 323 'last_build_number': 0, |
| 112 build_number, step_name, test_name) | 324 'lower_boundary': None, |
| 113 for _ in range(10): | 325 'upper_boundary': None, |
| 114 analysis.build_numbers.append(124) | 326 'lower_boundary_result': None, |
| 327 'sequential_run_index': 0 |
| 328 |
| 329 } |
| 330 self._CreateAndSaveMasterFlakeAnalysis( |
| 331 master_name, builder_name, build_number, step_name, |
| 332 test_name, status=analysis_status.PENDING |
| 333 ) |
| 334 self._CreateAndSaveFlakeSwarmingTask( |
| 335 master_name, builder_name, build_number, step_name, |
| 336 test_name, status=analysis_status.ERROR |
| 337 ) |
| 338 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 339 build_number, step_name, test_name) |
| 340 analysis.success_rates.append(.50) |
| 341 analysis.build_numbers.append(100) |
| 115 analysis.put() | 342 analysis.put() |
| 116 | 343 |
| 117 queue_name = {'x': False} | 344 queue_name = {'x': False} |
| 118 | 345 def my_mocked_run(arg1, queue_name): # pylint: disable=unused-argument |
| 119 # Unused argument (class method calls in python) - pylint: disable=W0613 | |
| 120 def my_mocked_run(*_): | |
| 121 queue_name['x'] = True # pragma: no cover | 346 queue_name['x'] = True # pragma: no cover |
| 122 | 347 |
| 123 self.mock( | 348 self.mock( |
| 124 recursive_flake_pipeline.RecursiveFlakePipeline, 'start', my_mocked_run) | 349 recursive_flake_pipeline.RecursiveFlakePipeline, 'start', my_mocked_run) |
| 125 NextBuildNumberPipeline.run( | 350 NextBuildNumberPipeline.run( |
| 126 NextBuildNumberPipeline(), master_name, builder_name, | 351 NextBuildNumberPipeline(), master_name, builder_name, |
| 127 master_build_number, step_name, test_name, test_result_future, | 352 master_build_number, build_number, step_name, test_name, |
| 128 queue_name) | 353 test_result_future, queue_name, flakiness_algorithm_results_dict) |
| 129 self.assertFalse(queue_name['x']) | 354 self.assertFalse(queue_name['x']) |
| 355 |
| 356 def testNextBuildPipelineForNewRecursionStabledFlakedOut(self): |
| 357 master_name = 'm' |
| 358 builder_name = 'b' |
| 359 master_build_number = 100 |
| 360 build_number = 100 |
| 361 step_name = 's' |
| 362 test_name = 't' |
| 363 test_result_future = 'trf' |
| 364 queue_name = constants.DEFAULT_QUEUE |
| 365 flakiness_algorithm_results_dict = { |
| 366 'flakes_in_a_row': 4, |
| 367 'stable_in_a_row': 0, |
| 368 'stabled_out': True, |
| 369 'flaked_out': False, |
| 370 'last_build_number': 0, |
| 371 'lower_boundary': 200, |
| 372 'upper_boundary': 210, |
| 373 'lower_boundary_result': 'FLAKE', |
| 374 'sequential_run_index': 0 |
| 375 } |
| 376 self._CreateAndSaveMasterFlakeAnalysis( |
| 377 master_name, builder_name, build_number, step_name, |
| 378 test_name, status=analysis_status.PENDING |
| 379 ) |
| 380 self._CreateAndSaveFlakeSwarmingTask( |
| 381 master_name, builder_name, build_number, step_name, |
| 382 test_name, status=analysis_status.COMPLETED |
| 383 ) |
| 384 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 385 build_number, step_name, test_name) |
| 386 analysis.success_rates.append(.50) |
| 387 analysis.build_numbers.append(100) |
| 388 analysis.put() |
| 389 |
| 390 queue_name = {'x': False} |
| 391 def my_mocked_run(arg1, queue_name): # pylint: disable=unused-argument |
| 392 queue_name['x'] = True # pragma: no cover |
| 393 |
| 394 self.mock( |
| 395 recursive_flake_pipeline.RecursiveFlakePipeline, 'start', my_mocked_run) |
| 396 NextBuildNumberPipeline.run( |
| 397 NextBuildNumberPipeline(), master_name, builder_name, |
| 398 master_build_number, build_number, step_name, test_name, |
| 399 test_result_future, queue_name, flakiness_algorithm_results_dict) |
| 400 self.assertTrue(queue_name['x']) |
| 401 |
| 402 def testGetNextRunSetStableLowerBoundary(self): |
| 403 master_name = 'm' |
| 404 builder_name = 'b' |
| 405 build_number = 100 |
| 406 step_name = 's' |
| 407 test_name = 't' |
| 408 self._CreateAndSaveMasterFlakeAnalysis( |
| 409 master_name, builder_name, build_number, step_name, |
| 410 test_name, status=analysis_status.PENDING |
| 411 ) |
| 412 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 413 build_number, step_name, test_name) |
| 414 analysis.success_rates.append(1) |
| 415 analysis.build_numbers.append(100) |
| 416 analysis.put() |
| 417 |
| 418 flakiness_algorithm_results_dict = { |
| 419 'flakes_in_a_row': 4, |
| 420 'stable_in_a_row': 0, |
| 421 'stabled_out': False, |
| 422 'flaked_out': True, |
| 423 'last_build_number': 0, |
| 424 'lower_boundary': None, |
| 425 'upper_boundary': 120, |
| 426 'lower_boundary_result': None, |
| 427 'sequential_run_index': 0 |
| 428 } |
| 429 get_next_run(analysis, flakiness_algorithm_results_dict) |
| 430 self.assertEqual(flakiness_algorithm_results_dict['lower_boundary'], |
| 431 build_number) |
| 432 self.assertEqual(flakiness_algorithm_results_dict['lower_boundary_result'], |
| 433 'STABLE') |
| 434 |
| 435 def testGetNextRunSetFlakeLowerBoundary(self): |
| 436 master_name = 'm' |
| 437 builder_name = 'b' |
| 438 build_number = 100 |
| 439 step_name = 's' |
| 440 test_name = 't' |
| 441 self._CreateAndSaveMasterFlakeAnalysis( |
| 442 master_name, builder_name, build_number, step_name, |
| 443 test_name, status=analysis_status.PENDING |
| 444 ) |
| 445 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 446 build_number, step_name, test_name) |
| 447 analysis.success_rates.append(.5) |
| 448 analysis.build_numbers.append(100) |
| 449 analysis.put() |
| 450 |
| 451 flakiness_algorithm_results_dict = { |
| 452 'flakes_in_a_row': 0, |
| 453 'stable_in_a_row': 0, |
| 454 'stabled_out': True, |
| 455 'flaked_out': False, |
| 456 'last_build_number': 0, |
| 457 'lower_boundary': None, |
| 458 'upper_boundary': None, |
| 459 'lower_boundary_result': None, |
| 460 'sequential_run_index': 0 |
| 461 } |
| 462 get_next_run(analysis, flakiness_algorithm_results_dict) |
| 463 self.assertEqual(flakiness_algorithm_results_dict['lower_boundary'], |
| 464 build_number) |
| 465 self.assertEqual(flakiness_algorithm_results_dict['lower_boundary_result'], |
| 466 'FLAKE') |
| 467 |
| 468 def testSequentialNextRunFirstTime(self): |
| 469 master_name = 'm' |
| 470 builder_name = 'b' |
| 471 build_number = 100 |
| 472 step_name = 's' |
| 473 test_name = 't' |
| 474 self._CreateAndSaveMasterFlakeAnalysis( |
| 475 master_name, builder_name, build_number, step_name, |
| 476 test_name, status=analysis_status.PENDING |
| 477 ) |
| 478 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 479 build_number, step_name, test_name) |
| 480 analysis.success_rates.append(.5) |
| 481 analysis.build_numbers.append(100) |
| 482 analysis.put() |
| 483 |
| 484 flakiness_algorithm_results_dict = { |
| 485 'flakes_in_a_row': 0, |
| 486 'stable_in_a_row': 0, |
| 487 'stabled_out': True, |
| 488 'flaked_out': True, |
| 489 'last_build_number': 0, |
| 490 'lower_boundary': 100, |
| 491 'upper_boundary': 110, |
| 492 'lower_boundary_result': 'STABLE', |
| 493 'sequential_run_index': 0 |
| 494 } |
| 495 next_run = sequential_next_run(analysis, flakiness_algorithm_results_dict) |
| 496 self.assertEqual(next_run, 101) |
| 497 |
| 498 def testSequentialFoundBorderFlake(self): |
| 499 master_name = 'm' |
| 500 builder_name = 'b' |
| 501 build_number = 100 |
| 502 step_name = 's' |
| 503 test_name = 't' |
| 504 self._CreateAndSaveMasterFlakeAnalysis( |
| 505 master_name, builder_name, build_number, step_name, |
| 506 test_name, status=analysis_status.PENDING |
| 507 ) |
| 508 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 509 build_number, step_name, test_name) |
| 510 analysis.success_rates.append(.5) |
| 511 analysis.build_numbers.append(100) |
| 512 analysis.put() |
| 513 |
| 514 flakiness_algorithm_results_dict = { |
| 515 'flakes_in_a_row': 0, |
| 516 'stable_in_a_row': 0, |
| 517 'stabled_out': True, |
| 518 'flaked_out': True, |
| 519 'last_build_number': 0, |
| 520 'lower_boundary': 100, |
| 521 'upper_boundary': 110, |
| 522 'lower_boundary_result': 'STABLE', |
| 523 'sequential_run_index': 1 |
| 524 } |
| 525 next_run = sequential_next_run(analysis, flakiness_algorithm_results_dict) |
| 526 self.assertEqual(next_run, False) |
| 527 self.assertEqual(analysis.suspected_flake_build_number, 101) |
| 528 |
| 529 def testSequentialFoundBorderStable(self): |
| 530 master_name = 'm' |
| 531 builder_name = 'b' |
| 532 build_number = 100 |
| 533 step_name = 's' |
| 534 test_name = 't' |
| 535 self._CreateAndSaveMasterFlakeAnalysis( |
| 536 master_name, builder_name, build_number, step_name, |
| 537 test_name, status=analysis_status.PENDING |
| 538 ) |
| 539 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 540 build_number, step_name, test_name) |
| 541 analysis.success_rates.append(1) |
| 542 analysis.build_numbers.append(100) |
| 543 analysis.put() |
| 544 |
| 545 flakiness_algorithm_results_dict = { |
| 546 'flakes_in_a_row': 0, |
| 547 'stable_in_a_row': 0, |
| 548 'stabled_out': True, |
| 549 'flaked_out': True, |
| 550 'last_build_number': 0, |
| 551 'lower_boundary': 100, |
| 552 'upper_boundary': 110, |
| 553 'lower_boundary_result': 'FLAKE', |
| 554 'sequential_run_index': 1 |
| 555 } |
| 556 next_run = sequential_next_run(analysis, flakiness_algorithm_results_dict) |
| 557 self.assertEqual(next_run, False) |
| 558 self.assertEqual(analysis.suspected_flake_build_number, 101) |
| 559 |
| 560 |
| 561 def testSequentialDidntFindBorderStable(self): |
| 562 master_name = 'm' |
| 563 builder_name = 'b' |
| 564 build_number = 100 |
| 565 step_name = 's' |
| 566 test_name = 't' |
| 567 self._CreateAndSaveMasterFlakeAnalysis( |
| 568 master_name, builder_name, build_number, step_name, |
| 569 test_name, status=analysis_status.PENDING |
| 570 ) |
| 571 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 572 build_number, step_name, test_name) |
| 573 analysis.success_rates.append(1) |
| 574 analysis.build_numbers.append(100) |
| 575 analysis.put() |
| 576 |
| 577 flakiness_algorithm_results_dict = { |
| 578 'flakes_in_a_row': 0, |
| 579 'stable_in_a_row': 0, |
| 580 'stabled_out': True, |
| 581 'flaked_out': True, |
| 582 'last_build_number': 0, |
| 583 'lower_boundary': 100, |
| 584 'upper_boundary': 110, |
| 585 'lower_boundary_result': 'STABLE', |
| 586 'sequential_run_index': 1 |
| 587 } |
| 588 next_run = sequential_next_run(analysis, flakiness_algorithm_results_dict) |
| 589 self.assertEqual(next_run, 102) |
| 590 self.assertEqual(analysis.suspected_flake_build_number, None) |
| 591 |
| 592 def testNextBuildPipelineStabledOutFlakedOutFirstTime(self): |
| 593 master_name = 'm' |
| 594 builder_name = 'b' |
| 595 master_build_number = 100 |
| 596 build_number = 100 |
| 597 step_name = 's' |
| 598 test_name = 't' |
| 599 test_result_future = 'trf' |
| 600 queue_name = constants.DEFAULT_QUEUE |
| 601 flakiness_algorithm_results_dict = { |
| 602 'flakes_in_a_row': 0, |
| 603 'stable_in_a_row': 0, |
| 604 'stabled_out': True, |
| 605 'flaked_out': True, |
| 606 'last_build_number': 0, |
| 607 'lower_boundary': 100, |
| 608 'upper_boundary': 110, |
| 609 'lower_boundary_result': None, |
| 610 'sequential_run_index': 0 |
| 611 |
| 612 } |
| 613 self._CreateAndSaveMasterFlakeAnalysis( |
| 614 master_name, builder_name, build_number, step_name, |
| 615 test_name, status=analysis_status.PENDING |
| 616 ) |
| 617 self._CreateAndSaveFlakeSwarmingTask( |
| 618 master_name, builder_name, build_number, step_name, |
| 619 test_name, status=analysis_status.COMPLETED |
| 620 ) |
| 621 analysis = MasterFlakeAnalysis.Get(master_name, builder_name, |
| 622 build_number, step_name, test_name) |
| 623 analysis.success_rates.append(1) |
| 624 analysis.build_numbers.append(50) |
| 625 analysis.put() |
| 626 |
| 627 NextBuildNumberPipeline.run( |
| 628 NextBuildNumberPipeline(), master_name, builder_name, |
| 629 master_build_number, build_number, step_name, test_name, |
| 630 test_result_future, queue_name, flakiness_algorithm_results_dict) |
| 631 self.assertEquals( |
| 632 flakiness_algorithm_results_dict['sequential_run_index'], 1) |
| OLD | NEW |