| OLD | NEW |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """This module is to provide Findit service APIs through Cloud Endpoints: | 5 """This module is to provide Findit service APIs through Cloud Endpoints: |
| 6 | 6 |
| 7 Current APIs include: | 7 Current APIs include: |
| 8 1. Analysis of compile/test failures in Chromium waterfalls. | 8 1. Analysis of compile/test failures in Chromium waterfalls. |
| 9 Analyzes failures and detects suspected CLs. | 9 Analyzes failures and detects suspected CLs. |
| 10 2. Analysis of flakes on Commit Queue. | 10 2. Analysis of flakes on Commit Queue. |
| (...skipping 141 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 152 queue_name=constants.WATERFALL_FLAKE_ANALYSIS_REQUEST_QUEUE) | 152 queue_name=constants.WATERFALL_FLAKE_ANALYSIS_REQUEST_QUEUE) |
| 153 | 153 |
| 154 | 154 |
| 155 # Create a Cloud Endpoints API. | 155 # Create a Cloud Endpoints API. |
| 156 # https://cloud.google.com/appengine/docs/python/endpoints/create_api | 156 # https://cloud.google.com/appengine/docs/python/endpoints/create_api |
| 157 @endpoints.api(name='findit', version='v1', description='FindIt API') | 157 @endpoints.api(name='findit', version='v1', description='FindIt API') |
| 158 class FindItApi(remote.Service): | 158 class FindItApi(remote.Service): |
| 159 """FindIt API v1.""" | 159 """FindIt API v1.""" |
| 160 | 160 |
| 161 def _GetConfidenceAndApproachForCL( | 161 def _GetConfidenceAndApproachForCL( |
| 162 self, repo_name, revision, confidences, build, first_failure): | 162 self, repo_name, revision, confidences, build, reference_build_key): |
| 163 cl = WfSuspectedCL.Get(repo_name, revision) | 163 cl = WfSuspectedCL.Get(repo_name, revision) |
| 164 if not cl: | 164 if not cl: |
| 165 return None, None | 165 return None, None |
| 166 | 166 |
| 167 master_name = buildbot.GetMasterNameFromUrl(build.master_url) | 167 master_name = buildbot.GetMasterNameFromUrl(build.master_url) |
| 168 builder_name = build.builder_name | 168 builder_name = build.builder_name |
| 169 current_build = build.build_number | 169 current_build = build.build_number |
| 170 | 170 |
| 171 # If the CL is found by a try job, only the first failure will be recorded. | 171 # If the CL is found by a try job, only the first failure will be recorded. |
| 172 # So we might need to go to the first failure to get CL information. | 172 # So we might need to go to the first failure to get CL information. |
| 173 build_info = cl.GetBuildInfo(master_name, builder_name, current_build) | 173 build_info = cl.GetBuildInfo(master_name, builder_name, current_build) |
| 174 first_build_info = cl.GetBuildInfo(master_name, builder_name, first_failure) | 174 first_build_info = cl.GetBuildInfo( |
| 175 *build_util.GetBuildInfoFromId(reference_build_key)) |
| 175 return suspected_cl_util.GetSuspectedCLConfidenceScoreAndApproach( | 176 return suspected_cl_util.GetSuspectedCLConfidenceScoreAndApproach( |
| 176 confidences, build_info, first_build_info) | 177 confidences, build_info, first_build_info) |
| 177 | 178 |
| 178 def _GenerateBuildFailureAnalysisResult( | 179 def _GenerateBuildFailureAnalysisResult( |
| 179 self, build, suspected_cls_in_result, step_name, first_failure, test_name, | 180 self, build, suspected_cls_in_result, step_name, first_failure, test_name, |
| 180 analysis_approach, confidences, try_job_status, is_flaky_test): | 181 analysis_approach, confidences, try_job_status, is_flaky_test, |
| 182 reference_build_key): |
| 181 | 183 |
| 182 suspected_cls = [] | 184 suspected_cls = [] |
| 183 for suspected_cl in suspected_cls_in_result: | 185 for suspected_cl in suspected_cls_in_result: |
| 184 repo_name = suspected_cl['repo_name'] | 186 repo_name = suspected_cl['repo_name'] |
| 185 revision = suspected_cl['revision'] | 187 revision = suspected_cl['revision'] |
| 186 commit_position = suspected_cl['commit_position'] | 188 commit_position = suspected_cl['commit_position'] |
| 187 confidence, cl_approach = self._GetConfidenceAndApproachForCL( | 189 confidence, cl_approach = self._GetConfidenceAndApproachForCL( |
| 188 repo_name, revision, confidences, build, first_failure) | 190 repo_name, revision, confidences, build, reference_build_key) |
| 189 if cl_approach: | 191 if cl_approach: |
| 190 cl_approach = ( | 192 cl_approach = ( |
| 191 _AnalysisApproach.HEURISTIC if | 193 _AnalysisApproach.HEURISTIC if |
| 192 cl_approach == analysis_approach_type.HEURISTIC else | 194 cl_approach == analysis_approach_type.HEURISTIC else |
| 193 _AnalysisApproach.TRY_JOB) | 195 _AnalysisApproach.TRY_JOB) |
| 194 else: | 196 else: |
| 195 cl_approach = analysis_approach | 197 cl_approach = analysis_approach |
| 196 | 198 |
| 197 suspected_cls.append(_SuspectedCL( | 199 suspected_cls.append(_SuspectedCL( |
| 198 repo_name=repo_name, revision=revision, | 200 repo_name=repo_name, revision=revision, |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 253 | 255 |
| 254 def _CheckIsFlaky(self, swarming_task, test_name): | 256 def _CheckIsFlaky(self, swarming_task, test_name): |
| 255 """Checks if the test is flaky.""" | 257 """Checks if the test is flaky.""" |
| 256 if not swarming_task or not swarming_task.classified_tests: | 258 if not swarming_task or not swarming_task.classified_tests: |
| 257 return False | 259 return False |
| 258 | 260 |
| 259 return test_name in swarming_task.classified_tests.get('flaky_tests', []) | 261 return test_name in swarming_task.classified_tests.get('flaky_tests', []) |
| 260 | 262 |
| 261 def _PopulateResult( | 263 def _PopulateResult( |
| 262 self, results, build, build_failure_type,heuristic_result, step_name, | 264 self, results, build, build_failure_type,heuristic_result, step_name, |
| 263 confidences, swarming_task, try_job, test_name=None): | 265 confidences, reference_build_key, swarming_task, try_job, test_name=None): |
| 264 """Appends an analysis result for the given step or test. | 266 """Appends an analysis result for the given step or test. |
| 265 | 267 |
| 266 Try-job results are always given priority over heuristic results. | 268 Try-job results are always given priority over heuristic results. |
| 267 """ | 269 """ |
| 268 # Default to heuristic analysis. | 270 # Default to heuristic analysis. |
| 269 suspected_cls = heuristic_result['suspected_cls'] | 271 suspected_cls = heuristic_result['suspected_cls'] |
| 270 analysis_approach = _AnalysisApproach.HEURISTIC | 272 analysis_approach = _AnalysisApproach.HEURISTIC |
| 271 | 273 |
| 272 # Check if the test is flaky. | 274 # Check if the test is flaky. |
| 273 is_flaky_test = self._CheckIsFlaky(swarming_task, test_name) | 275 is_flaky_test = self._CheckIsFlaky(swarming_task, test_name) |
| 274 | 276 |
| 275 if is_flaky_test: | 277 if is_flaky_test: |
| 276 suspected_cls = [] | 278 suspected_cls = [] |
| 277 try_job_status = _TryJobStatus.FINISHED # There will be no try job. | 279 try_job_status = _TryJobStatus.FINISHED # There will be no try job. |
| 278 else: | 280 else: |
| 279 # Check analysis result from try-job. | 281 # Check analysis result from try-job. |
| 280 try_job_status, culprit = self._GetStatusAndCulpritFromTryJob( | 282 try_job_status, culprit = self._GetStatusAndCulpritFromTryJob( |
| 281 try_job, swarming_task, build_failure_type, step_name, | 283 try_job, swarming_task, build_failure_type, step_name, |
| 282 test_name=test_name) | 284 test_name=test_name) |
| 283 if culprit: | 285 if culprit: |
| 284 suspected_cls = [culprit] | 286 suspected_cls = [culprit] |
| 285 analysis_approach = _AnalysisApproach.TRY_JOB | 287 analysis_approach = _AnalysisApproach.TRY_JOB |
| 286 if (not is_flaky_test and not suspected_cls and | 288 if (not is_flaky_test and not suspected_cls and |
| 287 not try_job_status == _TryJobStatus.RUNNING): | 289 not try_job_status == _TryJobStatus.RUNNING): |
| 288 return | 290 return |
| 289 | 291 |
| 290 results.append(self._GenerateBuildFailureAnalysisResult( | 292 results.append(self._GenerateBuildFailureAnalysisResult( |
| 291 build, suspected_cls, step_name, heuristic_result['first_failure'], | 293 build, suspected_cls, step_name, heuristic_result['first_failure'], |
| 292 test_name, analysis_approach, confidences, try_job_status, | 294 test_name, analysis_approach, confidences, try_job_status, |
| 293 is_flaky_test)) | 295 is_flaky_test, reference_build_key)) |
| 294 | 296 |
| 295 def _GetAllSwarmingTasks(self, failure_result_map): | 297 def _GetAllSwarmingTasks(self, failure_result_map): |
| 296 """Returns all swarming tasks related to one build. | 298 """Returns all swarming tasks related to one build. |
| 297 | 299 |
| 298 Args: | 300 Args: |
| 299 A dict to map each step/test with the key to the build when it failed the | 301 A dict to map each step/test with the key to the build when it failed the |
| 300 first time. | 302 first time. |
| 301 { | 303 { |
| 302 'step1': 'm/b/1', | 304 'step1': 'm/b/1', |
| 303 'step2': { | 305 'step2': { |
| (...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 367 for task_key in step_map.values(): | 369 for task_key in step_map.values(): |
| 368 if not try_jobs.get(task_key): | 370 if not try_jobs.get(task_key): |
| 369 try_jobs[task_key] = WfTryJob.Get(*task_key.split('/')) | 371 try_jobs[task_key] = WfTryJob.Get(*task_key.split('/')) |
| 370 | 372 |
| 371 return try_jobs | 373 return try_jobs |
| 372 | 374 |
| 373 def _GetSwarmingTaskAndTryJobForFailure( | 375 def _GetSwarmingTaskAndTryJobForFailure( |
| 374 self, step_name, test_name, failure_result_map, swarming_tasks, try_jobs): | 376 self, step_name, test_name, failure_result_map, swarming_tasks, try_jobs): |
| 375 """Gets swarming task and try job for the specific step/test.""" | 377 """Gets swarming task and try job for the specific step/test.""" |
| 376 if not failure_result_map: | 378 if not failure_result_map: |
| 377 return None, None | 379 return None, None, None |
| 378 | 380 |
| 379 if test_name: | 381 if test_name: |
| 380 try_job_key = failure_result_map.get(step_name, {}).get(test_name) | 382 try_job_key = failure_result_map.get(step_name, {}).get(test_name) |
| 381 else: | 383 else: |
| 382 try_job_key = failure_result_map.get(step_name) | 384 try_job_key = failure_result_map.get(step_name) |
| 383 | 385 |
| 384 # Gets the swarming task for the test. | 386 # Gets the swarming task for the test. |
| 385 swarming_task = swarming_tasks.get(step_name, {}).get(try_job_key) | 387 swarming_task = swarming_tasks.get(step_name, {}).get(try_job_key) |
| 386 | 388 |
| 387 # Get the try job for the step/test. | 389 # Get the try job for the step/test. |
| 388 try_job = try_jobs.get(try_job_key) | 390 try_job = try_jobs.get(try_job_key) |
| 389 | 391 |
| 390 return swarming_task, try_job | 392 return try_job_key, swarming_task, try_job |
| 391 | 393 |
| 392 def _GenerateResultsForBuild( | 394 def _GenerateResultsForBuild( |
| 393 self, build, heuristic_analysis, results, confidences): | 395 self, build, heuristic_analysis, results, confidences): |
| 394 | 396 |
| 395 swarming_tasks = self._GetAllSwarmingTasks( | 397 swarming_tasks = self._GetAllSwarmingTasks( |
| 396 heuristic_analysis.failure_result_map) | 398 heuristic_analysis.failure_result_map) |
| 397 try_jobs = self._GetAllTryJobs(heuristic_analysis.failure_result_map) | 399 try_jobs = self._GetAllTryJobs(heuristic_analysis.failure_result_map) |
| 398 | 400 |
| 399 for failure in heuristic_analysis.result['failures']: | 401 for failure in heuristic_analysis.result['failures']: |
| 400 step_name = failure['step_name'] | 402 step_name = failure['step_name'] |
| 401 if failure.get('tests'): # Test-level analysis. | 403 if failure.get('tests'): # Test-level analysis. |
| 402 for test in failure['tests']: | 404 for test in failure['tests']: |
| 403 test_name = test['test_name'] | 405 test_name = test['test_name'] |
| 404 swarming_task, try_job = self._GetSwarmingTaskAndTryJobForFailure( | 406 reference_build_key, swarming_task, try_job = ( |
| 405 step_name, test_name, heuristic_analysis.failure_result_map, | 407 self._GetSwarmingTaskAndTryJobForFailure( |
| 406 swarming_tasks, try_jobs) | 408 step_name, test_name, heuristic_analysis.failure_result_map, |
| 409 swarming_tasks, try_jobs)) |
| 407 | 410 |
| 408 self._PopulateResult( | 411 self._PopulateResult( |
| 409 results, build, heuristic_analysis.failure_type, test, | 412 results, build, heuristic_analysis.failure_type, test, |
| 410 step_name, confidences, swarming_task, try_job, | 413 step_name, confidences, reference_build_key, swarming_task, |
| 411 test_name=test_name) | 414 try_job, test_name=test_name) |
| 412 else: | 415 else: |
| 413 swarming_task, try_job = self._GetSwarmingTaskAndTryJobForFailure( | 416 reference_build_key, swarming_task, try_job = ( |
| 414 step_name, None, heuristic_analysis.failure_result_map, | 417 self._GetSwarmingTaskAndTryJobForFailure( |
| 415 swarming_tasks, try_jobs) | 418 step_name, None, heuristic_analysis.failure_result_map, |
| 419 swarming_tasks, try_jobs)) |
| 416 self._PopulateResult( | 420 self._PopulateResult( |
| 417 results, build, heuristic_analysis.failure_type, failure, | 421 results, build, heuristic_analysis.failure_type, failure, |
| 418 step_name, confidences, swarming_task, try_job) | 422 step_name, confidences, reference_build_key, swarming_task, try_job) |
| 419 | 423 |
| 420 @endpoints.method( | 424 @endpoints.method( |
| 421 _BuildFailureCollection, _BuildFailureAnalysisResultCollection, | 425 _BuildFailureCollection, _BuildFailureAnalysisResultCollection, |
| 422 path='buildfailure', name='buildfailure') | 426 path='buildfailure', name='buildfailure') |
| 423 def AnalyzeBuildFailures(self, request): | 427 def AnalyzeBuildFailures(self, request): |
| 424 """Returns analysis results for the given build failures in the request. | 428 """Returns analysis results for the given build failures in the request. |
| 425 | 429 |
| 426 Analysis of build failures will be triggered automatically on demand. | 430 Analysis of build failures will be triggered automatically on demand. |
| 427 | 431 |
| 428 Args: | 432 Args: |
| (...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 502 | 506 |
| 503 try: | 507 try: |
| 504 _AsyncProcessFlakeReport(flake_analysis_request, user_email, is_admin) | 508 _AsyncProcessFlakeReport(flake_analysis_request, user_email, is_admin) |
| 505 queued = True | 509 queued = True |
| 506 except Exception: | 510 except Exception: |
| 507 # Ignore the report when fail to queue it for async processing. | 511 # Ignore the report when fail to queue it for async processing. |
| 508 queued = False | 512 queued = False |
| 509 logging.exception('Failed to queue flake report for async processing') | 513 logging.exception('Failed to queue flake report for async processing') |
| 510 | 514 |
| 511 return _FlakeAnalysis(queued=queued) | 515 return _FlakeAnalysis(queued=queued) |
| OLD | NEW |