| OLD | NEW |
|---|
| (Empty) |
| 1 # coding=utf8 | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 """Sends patches to the Try server and reads back results. | |
| 6 | |
| 7 - RietveldTryJobs contains RietveldTryJob, one per try job on a builder. | |
| 8 - TryRunnerRietveld uses Rietveld to signal and poll job results. | |
| 9 """ | |
| 10 | |
| 11 import collections | |
| 12 import errno | |
| 13 import logging | |
| 14 import re | |
| 15 import socket | |
| 16 import time | |
| 17 import urllib2 | |
| 18 | |
| 19 import buildbot_json | |
| 20 import model | |
| 21 from verification import base | |
| 22 from verification import try_job_steps | |
| 23 | |
| 24 # A build running for longer than this is considered to be timed out. | |
| 25 TIMED_OUT = 12 * 60 * 60 | |
| 26 | |
| 27 | |
| 28 def is_job_expired(now, revision, timestamp, checkout): | |
| 29 """Returns False if the job result is still somewhat valid. | |
| 30 | |
| 31 A job that occured more than 4 days ago or more than 200 commits behind | |
| 32 is 'expired'. | |
| 33 """ | |
| 34 if timestamp < (now - 4*24*60*60): | |
| 35 return True | |
| 36 if checkout.revisions(revision, None) >= 200: | |
| 37 return True | |
| 38 return False | |
| 39 | |
| 40 | |
| 41 TryJobProperties = collections.namedtuple( | |
| 42 'TryJobProperties', | |
| 43 ['key', 'parent_key', 'builder', 'build', 'buildnumber', 'properties']) | |
| 44 | |
| 45 | |
| 46 def filter_jobs(try_job_results, watched_builders, current_irrelevant_keys, | |
| 47 status): | |
| 48 """For each try jobs results, query the Try Server for updated status and | |
| 49 returns details about each job in a TryJobProperties. | |
| 50 | |
| 51 Returns a list of namedtuple describing the updated results and and the new | |
| 52 list of irrelevant keys. | |
| 53 | |
| 54 It adds the build to the ignored list if the build doesn't exist on the Try | |
| 55 Server anymore (usually it's too old) or if the try job was not triggered by | |
| 56 the Commit Queue itself. | |
| 57 """ | |
| 58 irrelevant = set(current_irrelevant_keys) | |
| 59 try_jobs_with_props = [] | |
| 60 for result in try_job_results: | |
| 61 key = result['key'] | |
| 62 assert key | |
| 63 if key in current_irrelevant_keys: | |
| 64 continue | |
| 65 builder = result['builder'] | |
| 66 try: | |
| 67 buildnumber = int(result['buildnumber']) | |
| 68 except (TypeError, ValueError): | |
| 69 continue | |
| 70 if buildnumber < 0: | |
| 71 logging.debug('Ignoring %s/%d; invalid', builder, buildnumber) | |
| 72 irrelevant.add(key) | |
| 73 continue | |
| 74 | |
| 75 if builder not in watched_builders: | |
| 76 logging.debug('Ignoring %s/%d; no step verifier is examining it', builder, | |
| 77 buildnumber) | |
| 78 irrelevant.add(key) | |
| 79 continue | |
| 80 | |
| 81 # Constructing the object itself doesn't throw an exception, it's reading | |
| 82 # its properties that throws. | |
| 83 build = status.builders[builder].builds[buildnumber] | |
| 84 try: | |
| 85 props = build.properties_as_dict | |
| 86 except IOError: | |
| 87 logging.info( | |
| 88 'Build %s/%s is not on the try server anymore', | |
| 89 builder, buildnumber) | |
| 90 irrelevant.add(key) | |
| 91 continue | |
| 92 parent_key = props.get('parent_try_job_key') | |
| 93 if parent_key: | |
| 94 # Triggered build | |
| 95 key = '%s/%d_triggered_%s' % (builder, buildnumber, parent_key) | |
| 96 elif props.get('try_job_key') != key: | |
| 97 # not triggered, not valid | |
| 98 logging.debug( | |
| 99 'Ignoring %s/%d; not from rietveld', builder, buildnumber) | |
| 100 irrelevant.add(key) | |
| 101 continue | |
| 102 | |
| 103 try_jobs_with_props.append( | |
| 104 TryJobProperties(key, parent_key, builder, build, buildnumber, props)) | |
| 105 | |
| 106 # Sort the non-triggered builds first so triggered jobs | |
| 107 # can expect their parent to be added to self.try_jobs | |
| 108 try_jobs_with_props.sort(key=lambda tup: tup.parent_key) | |
| 109 | |
| 110 return try_jobs_with_props, list(irrelevant) | |
| 111 | |
| 112 | |
| 113 def _is_skip_try_job(pending): | |
| 114 """Returns True if a description contains NOTRY=true.""" | |
| 115 match = re.search(r'^NOTRY=(.*)$', pending.description, re.MULTILINE) | |
| 116 return match and match.group(1).lower() == 'true' | |
| 117 | |
| 118 | |
| 119 class RietveldTryJobPending(model.PersistentMixIn): | |
| 120 """Represents a pending try job for a pending commit that we care about. | |
| 121 | |
| 122 It is immutable. | |
| 123 """ | |
| 124 builder = unicode | |
| 125 revision = (None, unicode, int) | |
| 126 requested_steps = list | |
| 127 clobber = bool | |
| 128 # Number of retries for this configuration. Initial try is 1. | |
| 129 tries = int | |
| 130 init_time = float | |
| 131 | |
| 132 def __init__(self, **kwargs): | |
| 133 required = set(self._persistent_members()) | |
| 134 actual = set(kwargs) | |
| 135 assert required == actual, (required - actual, required, actual) | |
| 136 super(RietveldTryJobPending, self).__init__(**kwargs) | |
| 137 # Then mark it read-only. | |
| 138 self._read_only = True | |
| 139 | |
| 140 | |
| 141 class RietveldTryJob(model.PersistentMixIn): | |
| 142 """Represents a try job for a pending commit that we care about. | |
| 143 | |
| 144 This data can be regenerated by parsing all the try job names but it is a bit | |
| 145 hard on the try server. | |
| 146 | |
| 147 It is immutable. | |
| 148 """ | |
| 149 builder = unicode | |
| 150 build = int | |
| 151 revision = (None, unicode, int) | |
| 152 requested_steps = list | |
| 153 # The timestamp when the build started. buildbot_json returns int. | |
| 154 started = int | |
| 155 steps_passed = list | |
| 156 steps_failed = list | |
| 157 clobber = bool | |
| 158 completed = bool | |
| 159 # Number of retries for this configuration. Initial try is 1. | |
| 160 tries = int | |
| 161 parent_key = (None, unicode) | |
| 162 init_time = float | |
| 163 | |
| 164 def __init__(self, **kwargs): | |
| 165 required = set(self._persistent_members()) | |
| 166 actual = set(kwargs) | |
| 167 assert required == actual, (required - actual, required, actual) | |
| 168 super(RietveldTryJob, self).__init__(**kwargs) | |
| 169 # Then mark it read-only. | |
| 170 self._read_only = True | |
| 171 | |
| 172 @property | |
| 173 @model.immutable | |
| 174 def result(self): | |
| 175 if self.steps_failed: | |
| 176 return buildbot_json.FAILURE | |
| 177 if self.completed: | |
| 178 return buildbot_json.SUCCESS | |
| 179 return None | |
| 180 | |
| 181 | |
| 182 class RietveldTryJobs(base.IVerifierStatus): | |
| 183 """A set of try jobs that were sent for a specific patch. | |
| 184 | |
| 185 Multiple concurrent try jobs can be sent on a single builder. For example, a | |
| 186 previous valid try job could have been triggered by the user but was not | |
| 187 completed so another was sent with the missing tests. | |
| 188 Also, a try job is sent as soon as a test failure is detected. | |
| 189 """ | |
| 190 # An dict of RietveldTryJob objects per key. | |
| 191 try_jobs = dict | |
| 192 # The try job keys we ignore because they can't be used to give a good | |
| 193 # signal: either they are too old (old revision) or they were not triggerd | |
| 194 # by Rietveld, so we don't know if the diff is 100% good. | |
| 195 irrelevant = list | |
| 196 # When NOTRY=true is specified. | |
| 197 skipped = bool | |
| 198 # List of test verifiers. All the logic to decide when they are | |
| 199 # and what bots they trigger is hidden inside. | |
| 200 step_verifiers = list | |
| 201 # Jobs that have been sent but are not found yet. Likely a builder is fully | |
| 202 # utilized or the try server hasn't polled Rietveld yet. list of | |
| 203 # RietveldTryJobPending() instances. | |
| 204 pendings = list | |
| 205 | |
| 206 @model.immutable | |
| 207 def get_state(self): | |
| 208 """Returns the state of this verified. | |
| 209 | |
| 210 Failure can be from: | |
| 211 - For each entry in self.step_verifiers: | |
| 212 - A Try Job in self.try_jobs has been retried too often. | |
| 213 | |
| 214 In particular, there is no need to wait for every Try Job to complete. | |
| 215 """ | |
| 216 if self.error_message: | |
| 217 return base.FAILED | |
| 218 if not self.tests_waiting_for_result(): | |
| 219 return base.SUCCEEDED | |
| 220 return base.PROCESSING | |
| 221 | |
| 222 @model.immutable | |
| 223 def tests_need_to_be_run(self, now): | |
| 224 """Returns which tests need to be run. | |
| 225 | |
| 226 These are the tests that are not pending on any try job, either running or | |
| 227 in the pending list. | |
| 228 """ | |
| 229 # Skipped or failed, nothing to do. | |
| 230 if self.skipped or self.error_message: | |
| 231 return {} | |
| 232 | |
| 233 # What originally needed to be run. | |
| 234 # All_tests is {builder_name: set(test_name*)} | |
| 235 all_tests = {} | |
| 236 for verifier in self.step_verifiers: | |
| 237 (builder, tests) = verifier.need_to_trigger(self.try_jobs, now) | |
| 238 if tests: | |
| 239 all_tests.setdefault(builder, set()).update(tests) | |
| 240 | |
| 241 # Removes what is queued to be run but hasn't started yet. | |
| 242 for try_job in self.pendings: | |
| 243 if try_job.builder in all_tests: | |
| 244 all_tests[try_job.builder] -= set(try_job.requested_steps) | |
| 245 | |
| 246 return dict( | |
| 247 (builder, sorted(tests)) for builder, tests in all_tests.iteritems() | |
| 248 if tests) | |
| 249 | |
| 250 @model.immutable | |
| 251 def tests_waiting_for_result(self): | |
| 252 """Returns the tests that we are waiting for results on pending or running | |
| 253 builds. | |
| 254 """ | |
| 255 # Skipped or failed, nothing to do. | |
| 256 if self.skipped or self.error_message: | |
| 257 return {} | |
| 258 | |
| 259 # What originally needed to be run. | |
| 260 all_tests = {} | |
| 261 for verification in self.step_verifiers: | |
| 262 (builder, tests) = verification.waiting_for(self.try_jobs) | |
| 263 if tests: | |
| 264 all_tests.setdefault(builder, set()).update(tests) | |
| 265 | |
| 266 # Removes what was run. | |
| 267 for try_job in self.try_jobs.itervalues(): | |
| 268 if try_job.builder in all_tests: | |
| 269 all_tests[try_job.builder] -= set(try_job.steps_passed) | |
| 270 | |
| 271 return dict( | |
| 272 (builder, list(tests)) for builder, tests in all_tests.iteritems() | |
| 273 if tests) | |
| 274 | |
| 275 @model.immutable | |
| 276 def watched_builders(self): | |
| 277 """Marks all the jobs that the step_verifiers don't examine as | |
| 278 irrelevant. | |
| 279 """ | |
| 280 # Generate the list of builders to keep. | |
| 281 watched_builders = set() | |
| 282 for step_verifier in self.step_verifiers: | |
| 283 watched_builders.add(step_verifier.builder_name) | |
| 284 if isinstance(step_verifier, try_job_steps.TryJobTriggeredSteps): | |
| 285 watched_builders.add(step_verifier.trigger_name) | |
| 286 | |
| 287 return watched_builders | |
| 288 | |
| 289 def update_jobs_from_rietveld( | |
| 290 self, data, status, checkout, now): | |
| 291 """Retrieves the jobs statuses from rietveld and updates its state. | |
| 292 | |
| 293 Args: | |
| 294 owner: Owner of the CL. | |
| 295 data: Patchset properties as returned from Rietveld. | |
| 296 status: A buildbot_json.Buildbot instance. | |
| 297 checkout: A depot_tools' Checkout instance. | |
| 298 now: epoch time of what should be considered to be 'now'. | |
| 299 | |
| 300 Returns: | |
| 301 Keys which were updated. | |
| 302 """ | |
| 303 updated = [] | |
| 304 try_job_results = data.get('try_job_results', []) | |
| 305 logging.debug('Found %d entries', len(try_job_results)) | |
| 306 | |
| 307 try_jobs_with_props, self.irrelevant = filter_jobs( | |
| 308 try_job_results, self.watched_builders() , self.irrelevant, status) | |
| 309 | |
| 310 # Ensure that all irrelevant jobs have been removed from the set of valid | |
| 311 # try jobs. | |
| 312 for irrelevant_key in self.irrelevant: | |
| 313 if irrelevant_key in self.try_jobs: | |
| 314 del self.try_jobs[irrelevant_key] | |
| 315 if irrelevant_key + '_old' in self.try_jobs: | |
| 316 del self.try_jobs[irrelevant_key + '_old'] | |
| 317 | |
| 318 for i in try_jobs_with_props: | |
| 319 if self._update_try_job_status(checkout, i, now): | |
| 320 updated.append(i.key) | |
| 321 return updated | |
| 322 | |
| 323 def _update_try_job_status(self, checkout, try_job_properties, now): | |
| 324 """Updates status of a specific RietveldTryJob. | |
| 325 | |
| 326 try_job_property is an instance of TryJobProperties. | |
| 327 | |
| 328 Returns True if it was updated. | |
| 329 """ | |
| 330 key = try_job_properties.key | |
| 331 builder = try_job_properties.builder | |
| 332 buildnumber = try_job_properties.buildnumber | |
| 333 if key in self.irrelevant: | |
| 334 logging.debug('Ignoring %s/%d; irrelevant', builder, buildnumber) | |
| 335 return False | |
| 336 if (try_job_properties.parent_key and | |
| 337 try_job_properties.parent_key not in self.try_jobs): | |
| 338 logging.debug('Ignoring %s, parent unknown', key) | |
| 339 return False | |
| 340 | |
| 341 requested_steps = [] | |
| 342 # Set it to 0 as the default value since when the job is new and previous | |
| 343 # try jobs are found, we don't want to count them as tries. | |
| 344 tries = 0 | |
| 345 job = self.try_jobs.get(key) | |
| 346 build = try_job_properties.build | |
| 347 if job: | |
| 348 if job.completed: | |
| 349 logging.debug('Ignoring %s/%d; completed', builder, buildnumber) | |
| 350 return False | |
| 351 else: | |
| 352 if now - job.started > TIMED_OUT: | |
| 353 # Flush it and start over. | |
| 354 self.irrelevant.append(key) | |
| 355 del self.try_jobs[key] | |
| 356 return False | |
| 357 requested_steps = job.requested_steps | |
| 358 tries = job.tries | |
| 359 init_time = job.init_time | |
| 360 else: | |
| 361 # This try job is new. See if we triggered it previously by | |
| 362 # looking in self.pendings. | |
| 363 for index, pending_job in enumerate(self.pendings): | |
| 364 if pending_job.builder == builder: | |
| 365 # Reuse its item. | |
| 366 requested_steps = pending_job.requested_steps | |
| 367 tries = pending_job.tries | |
| 368 self.pendings.pop(index) | |
| 369 break | |
| 370 else: | |
| 371 # Is this a good build? It must not be too old and triggered by | |
| 372 # rietveld. | |
| 373 if is_job_expired(now, build.revision, build.start_time, checkout): | |
| 374 logging.debug('Ignoring %s/%d; expired', builder, buildnumber) | |
| 375 self.irrelevant.append(key) | |
| 376 return False | |
| 377 init_time = now | |
| 378 | |
| 379 passed = [s.name for s in build.steps if s.simplified_result] | |
| 380 failed = [s.name for s in build.steps if s.simplified_result is False] | |
| 381 # The steps in neither passed or failed were skipped. | |
| 382 new_job = RietveldTryJob( | |
| 383 init_time=init_time, | |
| 384 builder=builder, | |
| 385 build=buildnumber, | |
| 386 revision=build.revision, | |
| 387 requested_steps=requested_steps, | |
| 388 started=build.start_time, | |
| 389 steps_passed=passed, | |
| 390 steps_failed=failed, | |
| 391 clobber=bool(try_job_properties.properties.get('clobber')), | |
| 392 completed=build.completed, | |
| 393 tries=tries, | |
| 394 parent_key=try_job_properties.parent_key) | |
| 395 if job and job.build and new_job.build and job.build != new_job.build: | |
| 396 # It's tricky because 'key' is the same for both. The trick is to create | |
| 397 # a fake key for the old build and mark it as completed. Note that | |
| 398 # Rietveld is confused by it too. | |
| 399 logging.warning( | |
| 400 'Try Server was restarted and restarted builds with the same keys. ' | |
| 401 'I\'m confused. %s: %d != %d', job.builder, job.build, new_job.build) | |
| 402 # Resave the old try job and mark it as completed. | |
| 403 self.try_jobs[key + '_old'] = RietveldTryJob( | |
| 404 init_time=job.init_time, | |
| 405 builder=job.builder, | |
| 406 build=job.build, | |
| 407 revision=job.revision, | |
| 408 requested_steps=job.requested_steps, | |
| 409 started=build.start_time, | |
| 410 steps_passed=job.steps_passed, | |
| 411 steps_failed=job.steps_failed, | |
| 412 clobber=job.clobber, | |
| 413 completed=True, | |
| 414 tries=job.tries, | |
| 415 parent_key=job.parent_key) | |
| 416 if not job or not model.is_equivalent(new_job, job): | |
| 417 logging.info( | |
| 418 'Job update: %s: %s/%d', | |
| 419 try_job_properties.properties.get('issue'), | |
| 420 builder, | |
| 421 buildnumber) | |
| 422 self.try_jobs[key] = new_job | |
| 423 return key | |
| 424 | |
| 425 def signal_as_failed_if_needed(self, job, url, now): | |
| 426 """Detects if the RietveldTryJob instance is in a state where it is | |
| 427 impossible to make progress. | |
| 428 | |
| 429 If so, mark ourself as failed by setting self.error_message and return True. | |
| 430 """ | |
| 431 if self.skipped or self.error_message: | |
| 432 return False | |
| 433 # Figure out steps that should be retried for this builder. | |
| 434 missing_tests = self.tests_need_to_be_run(now).get(job.builder, []) | |
| 435 if not missing_tests: | |
| 436 return False | |
| 437 if job.tries > 2: | |
| 438 self.error_message = ( | |
| 439 'Retried try job too often on %s for step(s) %s\n%s' % | |
| 440 (job.builder, ', '.join(missing_tests), url)) | |
| 441 logging.info(self.error_message) | |
| 442 return True | |
| 443 return False | |
| 444 | |
| 445 @model.immutable | |
| 446 def why_not(self): | |
| 447 # Skipped or failed, nothing to do. | |
| 448 if self.skipped or self.error_message: | |
| 449 return None | |
| 450 waiting = self.tests_waiting_for_result() | |
| 451 if waiting: | |
| 452 out = 'Waiting for the following jobs:\n' | |
| 453 for builder in sorted(waiting): | |
| 454 out += ' %s: %s\n' % (builder, ','.join(waiting[builder])) | |
| 455 return out | |
| 456 | |
| 457 | |
| 458 class TryRunnerRietveld(base.VerifierCheckout): | |
| 459 """Stateless communication with a try server. | |
| 460 | |
| 461 Uses Rietveld to trigger the try job and reads try job status with the json | |
| 462 API. | |
| 463 | |
| 464 Analysis goes as following: | |
| 465 - compile step itself is not flaky. compile.py already takes care of most | |
| 466 flakiness and clobber build is done by default. If compile step fails, try | |
| 467 again with clobber=True | |
| 468 - test steps are flaky and can be retried as necessary. | |
| 469 | |
| 470 1. For each existing try jobs from rietveld. | |
| 471 1. Fetch result from try server. | |
| 472 2. If try job was generated from rietveld; | |
| 473 1. If not is_job_expired(); | |
| 474 1. Skip any scheduled test that succeeded on this builder. | |
| 475 2. For each builder with tests scheduled; | |
| 476 1. If no step waiting to be triggered, skip this builder completely. | |
| 477 2. For each non succeeded job; | |
| 478 1. Send try jobs to rietveld. | |
| 479 | |
| 480 Note: It needs rietveld, hence it uses VerifierCheckout, but it doesn't need a | |
| 481 checkout. | |
| 482 """ | |
| 483 name = 'try job rietveld' | |
| 484 | |
| 485 # Only updates a job status once every 60 seconds. | |
| 486 update_latency = 60 | |
| 487 | |
| 488 def __init__( | |
| 489 self, context_obj, try_server_url, commit_user, step_verifiers, | |
| 490 ignored_steps, solution): | |
| 491 super(TryRunnerRietveld, self).__init__(context_obj) | |
| 492 self.try_server_url = try_server_url.rstrip('/') | |
| 493 self.commit_user = commit_user | |
| 494 # TODO(maruel): Have it be overridden by presubmit_support.DoGetTrySlaves. | |
| 495 self.step_verifiers = step_verifiers | |
| 496 self.ignored_steps = set(ignored_steps) | |
| 497 # Time to poll the Try Server, and not Rietveld. | |
| 498 self.last_update = time.time() - self.update_latency | |
| 499 self.solution = solution | |
| 500 | |
| 501 def verify(self, pending): | |
| 502 """Sends a try job to the try server and returns a RietveldTryJob list. | |
| 503 | |
| 504 This function is called synchronously. | |
| 505 """ | |
| 506 jobs = pending.verifications.setdefault(self.name, RietveldTryJobs()) | |
| 507 if _is_skip_try_job(pending): | |
| 508 # Do not run try job for it. | |
| 509 jobs.skipped = True | |
| 510 return | |
| 511 | |
| 512 # Overridde any previous list from the last restart. | |
| 513 jobs.step_verifiers = [] | |
| 514 for step in self.step_verifiers: | |
| 515 if isinstance(step, try_job_steps.TryJobTriggeredOrNormalSteps): | |
| 516 # Since the steps are immutable, create a new step so that swarm | |
| 517 # can be enabled. | |
| 518 jobs.step_verifiers.append(try_job_steps.TryJobTriggeredOrNormalSteps( | |
| 519 builder_name=step.builder_name, | |
| 520 trigger_name=step.trigger_name, | |
| 521 steps=step.steps, | |
| 522 trigger_bot_steps=step.trigger_bot_steps, | |
| 523 use_triggered_bot=True)) | |
| 524 else: | |
| 525 jobs.step_verifiers.append(step) | |
| 526 | |
| 527 # First, update the status of the current try jobs on Rietveld. | |
| 528 now = time.time() | |
| 529 self._update_jobs_from_rietveld(pending, jobs, False, now) | |
| 530 | |
| 531 # Add anything that is missing. | |
| 532 self._send_jobs(pending, jobs, now) | |
| 533 | |
| 534 # Slightly postpone next check. | |
| 535 self.last_update = min(now, self.last_update + (self.update_latency / 4)) | |
| 536 | |
| 537 def update_status(self, queue): | |
| 538 """Grabs the current status of all try jobs and update self.queue. | |
| 539 | |
| 540 Note: it would be more efficient to be event based. | |
| 541 """ | |
| 542 if not queue: | |
| 543 logging.debug('The list is empty, nothing to do') | |
| 544 return | |
| 545 | |
| 546 # Hard code 'now' to the value before querying and sending them. This will | |
| 547 # cause some issues when querying state or sending the jobs takes a | |
| 548 # non-trivial amount of time but in general it will be fine. | |
| 549 now = time.time() | |
| 550 if now - self.last_update < self.update_latency: | |
| 551 logging.debug('TS: Throttling updates') | |
| 552 return | |
| 553 self.last_update = now | |
| 554 | |
| 555 # Update the status of the current pending CLs on Rietveld. | |
| 556 for pending, jobs in self.loop(queue, RietveldTryJobs, True): | |
| 557 # Update 'now' since querying the try jobs may take a significant amount | |
| 558 # of time. | |
| 559 now = time.time() | |
| 560 if self._update_jobs_from_rietveld(pending, jobs, True, now): | |
| 561 # Send any necessary job. Noop if not needed. | |
| 562 self._send_jobs(pending, jobs, now) | |
| 563 | |
| 564 def _add_pending_job_and_send_if_needed(self, builder, steps, jobs, | |
| 565 send_job, pending, now): | |
| 566 # Find if there was a previous try. | |
| 567 previous_jobs = [ | |
| 568 job for job in jobs.try_jobs.itervalues() if job.builder == builder | |
| 569 ] | |
| 570 if previous_jobs: | |
| 571 tries = max(job.tries for job in previous_jobs) | |
| 572 clobber = max( | |
| 573 (job.clobber or 'compile' in job.steps_failed) | |
| 574 for job in previous_jobs) | |
| 575 else: | |
| 576 tries = 0 | |
| 577 clobber = False | |
| 578 if tries > 4: | |
| 579 # Fail safe. | |
| 580 jobs.error_message = ( | |
| 581 ( 'The commit queue went berserk retrying too often for a\n' | |
| 582 'seemingly flaky test on builder %s:\n%s') % | |
| 583 ( builder, | |
| 584 '\n'.join(self._build_status_url(j) for j in previous_jobs))) | |
| 585 return False | |
| 586 | |
| 587 # Don't always send the job (triggered bots don't need to send there own | |
| 588 # request). | |
| 589 if send_job: | |
| 590 logging.debug( | |
| 591 'Sending job %s for %s: %s', pending.issue, builder, ','.join(steps)) | |
| 592 try: | |
| 593 self.context.rietveld.trigger_try_jobs( | |
| 594 pending.issue, pending.patchset, 'CQ', clobber, 'HEAD', | |
| 595 {builder: steps}) | |
| 596 except urllib2.HTTPError as e: | |
| 597 if e.code == 400: | |
| 598 # This probably mean a new patchset was uploaded since the last poll, | |
| 599 # so it's better to drop the CL. | |
| 600 jobs.error_message = 'Failed to trigger a try job on %s\n%s' % ( | |
| 601 builder, e) | |
| 602 return False | |
| 603 else: | |
| 604 raise | |
| 605 | |
| 606 # Set the status of this pending job here and on the CQ page. | |
| 607 jobs.pendings.append( | |
| 608 RietveldTryJobPending( | |
| 609 init_time=now, | |
| 610 builder=builder, | |
| 611 revision=None, | |
| 612 requested_steps=steps, | |
| 613 clobber=clobber, | |
| 614 tries=tries + 1)) | |
| 615 # Update the status on the AppEngine status to signal a new try job was | |
| 616 # sent. | |
| 617 info = { | |
| 618 'builder': builder, | |
| 619 'clobber': clobber, | |
| 620 'job_name': 'CQ', | |
| 621 'revision': None, #revision, | |
| 622 } | |
| 623 self.send_status(pending, info) | |
| 624 return True | |
| 625 | |
| 626 def _get_triggered_bots(self, builder, steps): | |
| 627 """Returns a dict of all the (builder, steps) pairs of bots that will get | |
| 628 triggered by the given builder steps combination.""" | |
| 629 triggered_bots = {} | |
| 630 for verifier in self.step_verifiers: | |
| 631 builder, steps = verifier.get_triggered_steps(builder, steps) | |
| 632 if steps: | |
| 633 triggered_bots[builder] = steps | |
| 634 | |
| 635 return triggered_bots | |
| 636 | |
| 637 def _send_jobs(self, pending, jobs, now): | |
| 638 """Prepares the RietveldTryJobs instance |jobs| to send try jobs to the try | |
| 639 server. | |
| 640 """ | |
| 641 if jobs.error_message: | |
| 642 # Too late. | |
| 643 return | |
| 644 remaining = jobs.tests_need_to_be_run(now) | |
| 645 if not remaining: | |
| 646 return | |
| 647 # Send them in order to simplify testing. | |
| 648 for builder in sorted(remaining): | |
| 649 tests = remaining[builder] | |
| 650 if not self._add_pending_job_and_send_if_needed(builder, tests, jobs, | |
| 651 True, pending, now): | |
| 652 # If the main job wasn't sent, we can't skip the triggered jobs since | |
| 653 # they won't get triggered. | |
| 654 continue | |
| 655 | |
| 656 # Add any pending bots that will be triggered from this build. | |
| 657 triggered_bots = self._get_triggered_bots(builder, tests) | |
| 658 for builder, steps in triggered_bots.iteritems(): | |
| 659 self._add_pending_job_and_send_if_needed(builder, steps, jobs, False, | |
| 660 pending, now) | |
| 661 | |
| 662 @model.immutable | |
| 663 def _build_status_url(self, job): | |
| 664 """Html url for this try job.""" | |
| 665 assert job.build is not None, str(job) | |
| 666 return '%s/buildstatus?builder=%s&number=%s' % ( | |
| 667 self.try_server_url, job.builder, job.build) | |
| 668 | |
| 669 @model.immutable | |
| 670 def _update_dashboard(self, pending, job): | |
| 671 """Updates the CQ dashboard with the current Try Job state as known to the | |
| 672 CQ. | |
| 673 """ | |
| 674 logging.debug('_update_dashboard(%s/%s)', job.builder, job.build) | |
| 675 info = { | |
| 676 'build': job.build, | |
| 677 'builder': job.builder, | |
| 678 'job_name': 'CQ', | |
| 679 'result': job.result, | |
| 680 'revision': job.revision, | |
| 681 'url': self._build_status_url(job), | |
| 682 } | |
| 683 self.send_status(pending, info) | |
| 684 | |
| 685 def _update_jobs_from_rietveld(self, pending, jobs, handle, now): | |
| 686 """Grabs data from Rietveld and pass it to | |
| 687 RietveldTryJobs.update_jobs_from_rietveld(). | |
| 688 | |
| 689 Returns True on success. | |
| 690 """ | |
| 691 status = buildbot_json.Buildbot(self.try_server_url) | |
| 692 try: | |
| 693 try: | |
| 694 data = self.context.rietveld.get_patchset_properties( | |
| 695 pending.issue, pending.patchset) | |
| 696 except urllib2.HTTPError as e: | |
| 697 if e.code == 404: | |
| 698 # TODO(phajdan.jr): Maybe generate a random id to correlate the user's | |
| 699 # error message and exception in the logs. | |
| 700 # Don't put exception traceback in the user-visible message to avoid | |
| 701 # leaking sensitive CQ data (passwords etc). | |
| 702 jobs.error_message = ('Failed to get patchset properties (patchset ' | |
| 703 'not found?)') | |
| 704 logging.error(str(e)) | |
| 705 return False | |
| 706 else: | |
| 707 raise | |
| 708 | |
| 709 # Update the RietvedTryJobs object. | |
| 710 keys = jobs.update_jobs_from_rietveld( | |
| 711 data, | |
| 712 status, | |
| 713 self.context.checkout, | |
| 714 now) | |
| 715 except urllib2.HTTPError as e: | |
| 716 if e.code in (500, 502, 503): | |
| 717 # Temporary AppEngine hiccup. Just log it and return failure. | |
| 718 logging.warning('%s while accessing %s. Ignoring error.' % ( | |
| 719 str(e), e.url)) | |
| 720 return False | |
| 721 else: | |
| 722 raise | |
| 723 except urllib2.URLError as e: | |
| 724 if 'timed out' in e.reason: | |
| 725 # Handle timeouts gracefully. | |
| 726 logging.warning('%s while updating tryserver status for ' | |
| 727 'rietveld issue %s', e, pending.issue) | |
| 728 return False | |
| 729 else: | |
| 730 raise | |
| 731 except socket.error as e: | |
| 732 # Temporary AppEngine hiccup. Just log it and return failure. | |
| 733 if e.errno == errno.ECONNRESET: | |
| 734 logging.warning( | |
| 735 '%s while updating tryserver status for rietveld issue %s.' % ( | |
| 736 str(e), str(pending.issue))) | |
| 737 return False | |
| 738 else: | |
| 739 raise | |
| 740 except IOError as e: | |
| 741 # Temporary AppEngine hiccup. Just log it and return failure. | |
| 742 if e.errno == 'socket error': | |
| 743 logging.warning( | |
| 744 '%s while updating tryserver status for rietveld issue %s.' % ( | |
| 745 str(e), str(pending.issue))) | |
| 746 return False | |
| 747 raise | |
| 748 | |
| 749 if handle: | |
| 750 for updated_key in keys: | |
| 751 job = jobs.try_jobs[updated_key] | |
| 752 self._update_dashboard(pending, job) | |
| 753 jobs.signal_as_failed_if_needed(job, self._build_status_url(job), now) | |
| 754 | |
| 755 return True | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 135363007:
Delete public commit queue to avoid confusion after move to internal repo (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/