| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 from datetime import datetime | 5 from datetime import datetime |
| 6 import logging | 6 import logging |
| 7 import textwrap | 7 import textwrap |
| 8 | 8 |
| 9 from google.appengine.ext import ndb | 9 from google.appengine.ext import ndb |
| 10 | 10 |
| 11 from common.git_repository import GitRepository | 11 from common.git_repository import GitRepository |
| 12 from common.http_client_appengine import HttpClientAppengine as HttpClient | 12 from common.http_client_appengine import HttpClientAppengine as HttpClient |
| 13 from common.pipeline_wrapper import BasePipeline | 13 from common.pipeline_wrapper import BasePipeline |
| 14 from common.rietveld import Rietveld | 14 from common.rietveld import Rietveld |
| 15 from model import analysis_status as status | 15 from model import analysis_status as status |
| 16 from model.wf_culprit import WfCulprit | 16 from model.wf_culprit import WfCulprit |
| 17 from waterfall import build_util |
| 17 from waterfall import waterfall_config | 18 from waterfall import waterfall_config |
| 18 | 19 |
| 19 | 20 |
| 20 @ndb.transactional | 21 @ndb.transactional |
| 21 def _ShouldSendNotification( | 22 def _ShouldSendNotification( |
| 22 master_name, builder_name, build_number, repo_name, revision, | 23 master_name, builder_name, build_number, repo_name, revision, |
| 23 build_num_threshold, time_limit_passed): | 24 commit_position, build_num_threshold, time_limit_passed): |
| 24 """Returns True if a notification for the culprit should be sent.""" | 25 """Returns True if a notification for the culprit should be sent.""" |
| 25 culprit = (WfCulprit.Get(repo_name, revision) or | 26 culprit = (WfCulprit.Get(repo_name, revision) or |
| 26 WfCulprit.Create(repo_name, revision)) | 27 WfCulprit.Create(repo_name, revision, commit_position)) |
| 27 if [master_name, builder_name, build_number] in culprit.builds: | 28 if [master_name, builder_name, build_number] in culprit.builds: |
| 28 return False | 29 return False |
| 29 | 30 |
| 30 culprit.builds.append([master_name, builder_name, build_number]) | 31 culprit.builds.append([master_name, builder_name, build_number]) |
| 31 # Send notification only when: | 32 # Send notification only when: |
| 32 # 1. It was not processed yet. | 33 # 1. It was not processed yet. |
| 33 # 2. The culprit is for multiple failures in different builds to avoid false | 34 # 2. The culprit is for multiple failures in different builds to avoid false |
| 34 # positive due to flakiness. | 35 # positive due to flakiness. |
| 35 # 3. It is not too late after the culprit was committed. | 36 # 3. It is not too late after the culprit was committed. |
| 36 # * Try-job takes too long to complete and the failure got fixed. | 37 # * Try-job takes too long to complete and the failure got fixed. |
| 37 # * The whole analysis was rerun a long time after the failure occurred. | 38 # * The whole analysis was rerun a long time after the failure occurred. |
| 38 should_send = not (culprit.cr_notification_processed or | 39 should_send = not (culprit.cr_notification_processed or |
| 39 len(culprit.builds) < build_num_threshold or | 40 len(culprit.builds) < build_num_threshold or |
| 40 time_limit_passed) | 41 time_limit_passed) |
| 41 if should_send: | 42 if should_send: |
| 42 culprit.cr_notification_status = status.RUNNING | 43 culprit.cr_notification_status = status.RUNNING |
| 43 culprit.put() | 44 culprit.put() |
| 44 return should_send | 45 return should_send |
| 45 | 46 |
| 46 | 47 |
| 47 @ndb.transactional | 48 @ndb.transactional |
| 48 def _UpdateNotificationStatus(repo_name, revision, new_status): | 49 def _UpdateNotificationStatus(repo_name, revision, new_status): |
| 49 culprit = WfCulprit.Get(repo_name, revision) | 50 culprit = WfCulprit.Get(repo_name, revision) |
| 50 culprit.cr_notification_status = new_status | 51 culprit.cr_notification_status = new_status |
| 51 if culprit.cr_notified: | 52 if culprit.cr_notified: |
| 52 culprit.cr_notification_time = datetime.utcnow() | 53 culprit.cr_notification_time = datetime.utcnow() |
| 53 culprit.put() | 54 culprit.put() |
| 54 | 55 |
| 55 | 56 |
| 56 def _SendNotificationForCulprit(repo_name, revision, code_review_url): | 57 def _SendNotificationForCulprit( |
| 58 repo_name, revision, commit_position, code_review_url): |
| 57 sent = False | 59 sent = False |
| 58 if code_review_url: | 60 if code_review_url: |
| 59 # Occasionally, a commit was not uploaded for code-review. | 61 # Occasionally, a commit was not uploaded for code-review. |
| 60 culprit = WfCulprit.Get(repo_name, revision) | 62 culprit = WfCulprit.Get(repo_name, revision) |
| 61 rietveld = Rietveld() | 63 rietveld = Rietveld() |
| 62 message = textwrap.dedent(""" | 64 message = textwrap.dedent(""" |
| 63 Findit Try-job identified this CL (revision %s) as the culprit for failures | 65 FYI: Findit try jobs (rerunning failed compile or tests) identified this CL |
| 64 in the build cycle(s) as shown on: | 66 at revision %s as the culprit for failures in the build cycles as shown on: |
| 65 https://findit-for-me.appspot.com/waterfall/culprit?key=%s | 67 https://findit-for-me.appspot.com/waterfall/culprit?key=%s |
| 66 """) % (revision, culprit.key.urlsafe()) | 68 """) % (commit_position or revision, culprit.key.urlsafe()) |
| 67 sent = rietveld.PostMessage(code_review_url, message) | 69 sent = rietveld.PostMessage(code_review_url, message) |
| 68 else: | 70 else: |
| 69 logging.error('No code-review url for %s/%s', repo_name, revision) | 71 logging.error('No code-review url for %s/%s', repo_name, revision) |
| 70 | 72 |
| 71 _UpdateNotificationStatus(repo_name, revision, | 73 _UpdateNotificationStatus(repo_name, revision, |
| 72 status.COMPLETED if sent else status.ERROR) | 74 status.COMPLETED if sent else status.ERROR) |
| 73 return sent | 75 return sent |
| 74 | 76 |
| 75 | 77 |
| 76 def _GetCulpritInfo(repo_name, revision): | 78 def _GetCulpritInfo(repo_name, revision): |
| 77 """Returns commit time and code-review url of the given revision.""" | 79 """Returns commit position/time and code-review url of the given revision.""" |
| 78 # TODO(stgao): get repo url at runtime based on the given repo name. | 80 # TODO(stgao): get repo url at runtime based on the given repo name. |
| 79 # unused arg - pylint: disable=W0612,W0613 | 81 # unused arg - pylint: disable=W0612,W0613 |
| 80 repo = GitRepository( | 82 repo = GitRepository( |
| 81 'https://chromium.googlesource.com/chromium/src.git', HttpClient()) | 83 'https://chromium.googlesource.com/chromium/src.git', HttpClient()) |
| 82 change_log = repo.GetChangeLog(revision) | 84 change_log = repo.GetChangeLog(revision) |
| 83 return change_log.committer_time, change_log.code_review_url | 85 return change_log.commit_position, change_log.code_review_url |
| 84 | 86 |
| 85 | 87 |
| 86 def _NotificationTimeLimitPassed(commit_time, latency_limit_minutes): | 88 def _NotificationTimeLimitPassed(build_end_time, latency_limit_minutes): |
| 87 """Returns True if it is too late to send notification.""" | 89 """Returns True if it is too late to send notification.""" |
| 88 latency_seconds = (datetime.utcnow() - commit_time).total_seconds() | 90 latency_seconds = (datetime.utcnow() - build_end_time).total_seconds() |
| 89 return latency_seconds > latency_limit_minutes * 60 | 91 return latency_seconds > latency_limit_minutes * 60 |
| 90 | 92 |
| 91 | 93 |
| 92 class SendNotificationForCulpritPipeline(BasePipeline): | 94 class SendNotificationForCulpritPipeline(BasePipeline): |
| 93 | 95 |
| 94 # Arguments number differs from overridden method - pylint: disable=W0221 | 96 # Arguments number differs from overridden method - pylint: disable=W0221 |
| 95 def run(self, master_name, builder_name, build_number, repo_name, revision): | 97 def run(self, master_name, builder_name, build_number, repo_name, revision): |
| 96 action_settings = waterfall_config.GetActionSettings() | 98 action_settings = waterfall_config.GetActionSettings() |
| 97 # Set some impossible default values to prevent notification by default. | 99 # Set some impossible default values to prevent notification by default. |
| 98 build_threshold = action_settings.get( | 100 build_threshold = action_settings.get( |
| 99 'cr_notification_build_threshold', 100000) | 101 'cr_notification_build_threshold', 100000) |
| 100 latency_limit_minutes = action_settings.get( | 102 latency_limit_minutes = action_settings.get( |
| 101 'cr_notification_latency_limit_minutes', 1) | 103 'cr_notification_latency_limit_minutes', 1) |
| 102 | 104 |
| 103 commit_time, code_review_url = _GetCulpritInfo(repo_name, revision) | 105 commit_position, code_review_url = _GetCulpritInfo( |
| 106 repo_name, revision) |
| 107 build_end_time = build_util.GetBuildEndTime( |
| 108 master_name, builder_name, build_number) |
| 104 time_limit_passed = _NotificationTimeLimitPassed( | 109 time_limit_passed = _NotificationTimeLimitPassed( |
| 105 commit_time, latency_limit_minutes) | 110 build_end_time, latency_limit_minutes) |
| 106 | 111 |
| 107 if not _ShouldSendNotification( | 112 if not _ShouldSendNotification( |
| 108 master_name, builder_name, build_number, | 113 master_name, builder_name, build_number, repo_name, |
| 109 repo_name, revision, build_threshold, time_limit_passed): | 114 revision, commit_position, build_threshold, time_limit_passed): |
| 110 return False | 115 return False |
| 111 return _SendNotificationForCulprit(repo_name, revision, code_review_url) | 116 return _SendNotificationForCulprit( |
| 117 repo_name, revision, commit_position, code_review_url) |
| OLD | NEW |