OLD | NEW |
1 # coding=utf8 | 1 # coding=utf8 |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 """Commit queue manager class. | 5 """Commit queue manager class. |
6 | 6 |
7 Security implications: | 7 Security implications: |
8 | 8 |
9 The following hypothesis are made: | 9 The following hypothesis are made: |
10 - Commit queue: | 10 - Commit queue: |
(...skipping 179 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
190 | 190 |
191 raise | 191 raise |
192 | 192 |
193 # If there is an issue in processed_issues that is not in new_issues, | 193 # If there is an issue in processed_issues that is not in new_issues, |
194 # discard it. | 194 # discard it. |
195 for pending in self.queue.iterate(): | 195 for pending in self.queue.iterate(): |
196 # Note that pending.issue is a int but self.queue.pending_commits keys | 196 # Note that pending.issue is a int but self.queue.pending_commits keys |
197 # are str due to json support. | 197 # are str due to json support. |
198 if pending.issue not in new_issues: | 198 if pending.issue not in new_issues: |
199 logging.info('Flushing issue %d' % pending.issue) | 199 logging.info('Flushing issue %d' % pending.issue) |
| 200 self.context.status.send( |
| 201 pending, |
| 202 { 'verification': 'abort', |
| 203 'payload': { |
| 204 'output': 'CQ bit was unchecked on CL. Ignoring.' }}) |
200 pending.get_state = lambda: base.IGNORED | 205 pending.get_state = lambda: base.IGNORED |
201 self._discard_pending(pending, 'CQ bit was unchecked on CL. Ignoring.') | 206 self._discard_pending(pending, None) |
202 | 207 |
203 # Find new issues. | 208 # Find new issues. |
204 for issue_id in new_issues: | 209 for issue_id in new_issues: |
205 if str(issue_id) not in self.queue.pending_commits: | 210 if str(issue_id) not in self.queue.pending_commits: |
206 try: | 211 try: |
207 issue_data = self.context.rietveld.get_issue_properties( | 212 issue_data = self.context.rietveld.get_issue_properties( |
208 issue_id, True) | 213 issue_id, True) |
209 except urllib2.HTTPError as e: | 214 except urllib2.HTTPError as e: |
210 if e.code in (500, 502, 503): | 215 if e.code in (500, 502, 503): |
211 # Temporary AppEngine hiccup. Just log it and continue. | 216 # Temporary AppEngine hiccup. Just log it and continue. |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
259 # Take in account the case where a verifier was removed. | 264 # Take in account the case where a verifier was removed. |
260 done = set(pending.verifications.keys()) | 265 done = set(pending.verifications.keys()) |
261 missing = expected - done | 266 missing = expected - done |
262 if (not missing or pending.get_state() != base.PROCESSING): | 267 if (not missing or pending.get_state() != base.PROCESSING): |
263 continue | 268 continue |
264 logging.info( | 269 logging.info( |
265 'Processing issue %s (%s, %d)' % ( | 270 'Processing issue %s (%s, %d)' % ( |
266 pending.issue, missing, pending.get_state())) | 271 pending.issue, missing, pending.get_state())) |
267 self._verify_pending(pending) | 272 self._verify_pending(pending) |
268 except base.DiscardPending as e: | 273 except base.DiscardPending as e: |
269 message = e.status | 274 self._discard_pending(e.pending, e.status) |
270 if not message: | |
271 message = 'process_new_pending_commit: ' + self.FAILED_NO_MESSAGE | |
272 self._discard_pending(e.pending, message) | |
273 | 275 |
274 def update_status(self): | 276 def update_status(self): |
275 """Updates the status for each pending commit verifier.""" | 277 """Updates the status for each pending commit verifier.""" |
276 why_nots = dict((p.issue, p.why_not()) for p in self.queue.iterate()) | 278 why_nots = dict((p.issue, p.why_not()) for p in self.queue.iterate()) |
277 | 279 |
278 for verifier in self.all_verifiers: | 280 for verifier in self.all_verifiers: |
279 try: | 281 try: |
280 verifier.update_status(self.queue.iterate()) | 282 verifier.update_status(self.queue.iterate()) |
281 except base.DiscardPending as e: | 283 except base.DiscardPending as e: |
282 # It's not efficient since it takes a full loop for each pending | 284 # It's not efficient since it takes a full loop for each pending |
283 # commit to discard. | 285 # commit to discard. |
284 message = e.status | 286 self._discard_pending(e.pending, e.status) |
285 if not message: | |
286 message = 'update_status: ' + self.FAILED_NO_MESSAGE | |
287 self._discard_pending(e.pending, message) | |
288 | 287 |
289 for pending in self.queue.iterate(): | 288 for pending in self.queue.iterate(): |
290 why_not = pending.why_not() | 289 why_not = pending.why_not() |
291 if why_nots[pending.issue] != why_not: | 290 if why_nots[pending.issue] != why_not: |
292 self.context.status.send( | 291 self.context.status.send( |
293 pending, | 292 pending, |
294 {'verification': 'why not', | 293 {'verification': 'why not', |
295 'payload': {'message': why_not}}) | 294 'payload': {'message': why_not}}) |
296 | 295 |
297 | 296 |
298 def scan_results(self): | 297 def scan_results(self): |
299 """Scans pending commits that can be committed or discarded.""" | 298 """Scans pending commits that can be committed or discarded.""" |
300 for pending in self.queue.iterate(): | 299 for pending in self.queue.iterate(): |
301 state = pending.get_state() | 300 state = pending.get_state() |
302 if state == base.FAILED: | 301 if state == base.FAILED: |
303 message = pending.error_message() | 302 self._discard_pending( |
304 if not message: | 303 pending, pending.error_message() or self.FAILED_NO_MESSAGE) |
305 message = 'scan_results(FAILED): ' + self.FAILED_NO_MESSAGE | |
306 self._discard_pending(pending, message) | |
307 elif state == base.SUCCEEDED: | 304 elif state == base.SUCCEEDED: |
308 if self._throttle(pending): | 305 if self._throttle(pending): |
309 continue | 306 continue |
310 try: | 307 try: |
311 # Runs checks. It's be nice to run the test before the postpone, | 308 # Runs checks. It's be nice to run the test before the postpone, |
312 # especially if the tree is closed for a long moment but at the same | 309 # especially if the tree is closed for a long moment but at the same |
313 # time it would keep fetching the rietveld status constantly. | 310 # time it would keep fetching the rietveld status constantly. |
314 self._last_minute_checks(pending) | 311 self._last_minute_checks(pending) |
315 self.context.status.send( | 312 self.context.status.send( |
316 pending, | 313 pending, |
317 {'verification': 'why not', | 314 {'verification': 'why not', |
318 'payload': {'message': ''}}) | 315 'payload': {'message': ''}}) |
319 | 316 |
320 self._commit_patch(pending) | 317 self._commit_patch(pending) |
321 except base.DiscardPending as e: | 318 except base.DiscardPending as e: |
322 message = e.status | 319 self._discard_pending(e.pending, e.status) |
323 if not message: | |
324 message = 'scan_results(discard): ' + self.FAILED_NO_MESSAGE | |
325 self._discard_pending(e.pending, message) | |
326 except Exception as e: | 320 except Exception as e: |
327 message = 'scan_result(Exception): ' + self.INTERNAL_EXCEPTION | 321 self._discard_pending(pending, self.INTERNAL_EXCEPTION) |
328 self._discard_pending(pending, message) | |
329 raise | 322 raise |
330 else: | 323 else: |
331 # When state is IGNORED, we need to keep this issue so it's not fetched | 324 # When state is IGNORED, we need to keep this issue so it's not fetched |
332 # another time but we can't discard it since we don't want to remove the | 325 # another time but we can't discard it since we don't want to remove the |
333 # commit bit for another project hosted on the same code review | 326 # commit bit for another project hosted on the same code review |
334 # instance. | 327 # instance. |
335 assert state in (base.PROCESSING, base.IGNORED) | 328 assert state in (base.PROCESSING, base.IGNORED) |
336 | 329 |
337 def _verify_pending(self, pending): | 330 def _verify_pending(self, pending): |
338 """Initiates all the verifiers on a pending change.""" | 331 """Initiates all the verifiers on a pending change.""" |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
423 pending, | 416 pending, |
424 'List of reviewers changed. %s did a drive-by without LGTM\'ing!' % | 417 'List of reviewers changed. %s did a drive-by without LGTM\'ing!' % |
425 ','.join(drivers_by)) | 418 ','.join(drivers_by)) |
426 if pending.patchset != pending_data['patchsets'][-1]: | 419 if pending.patchset != pending_data['patchsets'][-1]: |
427 raise base.DiscardPending(pending, | 420 raise base.DiscardPending(pending, |
428 'Commit queue failed due to new patchset.') | 421 'Commit queue failed due to new patchset.') |
429 | 422 |
430 def _discard_pending(self, pending, message): | 423 def _discard_pending(self, pending, message): |
431 """Discards a pending commit. Attach an optional message to the review.""" | 424 """Discards a pending commit. Attach an optional message to the review.""" |
432 logging.debug('_discard_pending(%s, %s)', pending.issue, message) | 425 logging.debug('_discard_pending(%s, %s)', pending.issue, message) |
433 msg = message or self.FAILED_NO_MESSAGE | |
434 try: | 426 try: |
435 try: | 427 try: |
436 if pending.get_state() != base.IGNORED: | 428 if pending.get_state() != base.IGNORED: |
437 self.context.rietveld.set_flag( | 429 self.context.rietveld.set_flag( |
438 pending.issue, pending.patchset, 'commit', False) | 430 pending.issue, pending.patchset, 'commit', False) |
439 except urllib2.HTTPError as e: | 431 except urllib2.HTTPError as e: |
440 logging.error( | 432 logging.error( |
441 'Failed to set the flag to False for %s with message %s' % ( | 433 'Failed to set the flag to False for %s with message %s' % ( |
442 pending.pending_name(), msg)) | 434 pending.pending_name(), message)) |
443 traceback.print_stack() | 435 traceback.print_stack() |
444 logging.error(str(e)) | 436 logging.error(str(e)) |
445 errors.send_stack(e) | 437 errors.send_stack(e) |
446 try: | 438 if message: |
447 self.context.rietveld.add_comment(pending.issue, msg) | 439 try: |
448 except urllib2.HTTPError as e: | 440 self.context.rietveld.add_comment(pending.issue, message) |
449 logging.error( | 441 except urllib2.HTTPError as e: |
450 'Failed to add comment for %s with message %s' % ( | 442 logging.error( |
451 pending.pending_name(), msg)) | 443 'Failed to add comment for %s with message %s' % ( |
452 traceback.print_stack() | 444 pending.pending_name(), message)) |
453 errors.send_stack(e) | 445 traceback.print_stack() |
454 self.context.status.send( | 446 errors.send_stack(e) |
455 pending, | 447 self.context.status.send( |
456 { 'verification': 'abort', | 448 pending, |
457 'payload': { | 449 { 'verification': 'abort', |
458 'output': msg }}) | 450 'payload': { |
| 451 'output': message }}) |
459 finally: | 452 finally: |
460 # Most importantly, remove the PendingCommit from the queue. | 453 # Most importantly, remove the PendingCommit from the queue. |
461 self.queue.remove(pending.issue) | 454 self.queue.remove(pending.issue) |
462 | 455 |
463 def _commit_patch(self, pending): | 456 def _commit_patch(self, pending): |
464 """Commits the pending patch to the repository. | 457 """Commits the pending patch to the repository. |
465 | 458 |
466 Do the checkout and applies the patch. | 459 Do the checkout and applies the patch. |
467 """ | 460 """ |
468 try: | 461 try: |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
527 except ( | 520 except ( |
528 checkout.PatchApplicationFailed, patch.UnsupportedPatchFormat) as e: | 521 checkout.PatchApplicationFailed, patch.UnsupportedPatchFormat) as e: |
529 raise base.DiscardPending(pending, str(e)) | 522 raise base.DiscardPending(pending, str(e)) |
530 except subprocess2.CalledProcessError as e: | 523 except subprocess2.CalledProcessError as e: |
531 stdout = getattr(e, 'stdout', None) | 524 stdout = getattr(e, 'stdout', None) |
532 out = 'Failed to apply the patch.' | 525 out = 'Failed to apply the patch.' |
533 if stdout: | 526 if stdout: |
534 out += '\n%s' % stdout | 527 out += '\n%s' % stdout |
535 raise base.DiscardPending(pending, out) | 528 raise base.DiscardPending(pending, out) |
536 except base.DiscardPending as e: | 529 except base.DiscardPending as e: |
537 message = e.status | 530 self._discard_pending(e.pending, e.status) |
538 if not message: | |
539 message = '_commit_patch: ' + self.FAILED_NO_MESSAGE | |
540 self._discard_pending(e.pending, message) | |
541 | 531 |
542 def _throttle(self, pending): | 532 def _throttle(self, pending): |
543 """Returns True if a commit should be delayed.""" | 533 """Returns True if a commit should be delayed.""" |
544 if pending.postpone(): | 534 if pending.postpone(): |
545 self.context.status.send( | 535 self.context.status.send( |
546 pending, | 536 pending, |
547 {'verification': 'why not', | 537 {'verification': 'why not', |
548 'payload': { | 538 'payload': { |
549 'message': pending.why_not()}}) | 539 'message': pending.why_not()}}) |
550 return True | 540 return True |
(...skipping 19 matching lines...) Expand all Loading... |
570 """Loads the commit queue state from a JSON file.""" | 560 """Loads the commit queue state from a JSON file.""" |
571 self.queue = model.load_from_json_file(filename) | 561 self.queue = model.load_from_json_file(filename) |
572 | 562 |
573 def save(self, filename): | 563 def save(self, filename): |
574 """Save the commit queue state in a simple JSON file.""" | 564 """Save the commit queue state in a simple JSON file.""" |
575 model.save_to_json_file(filename, self.queue) | 565 model.save_to_json_file(filename, self.queue) |
576 | 566 |
577 def close(self): | 567 def close(self): |
578 """Close all the active pending manager items.""" | 568 """Close all the active pending manager items.""" |
579 self.context.status.close() | 569 self.context.status.close() |
OLD | NEW |