| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Unit tests for verification/try_server.py.""" | |
| 7 | |
| 8 import json | |
| 9 import logging | |
| 10 import optparse | |
| 11 import os | |
| 12 import re | |
| 13 import StringIO | |
| 14 import sys | |
| 15 import time | |
| 16 import unittest | |
| 17 import urllib | |
| 18 | |
| 19 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| 20 sys.path.insert(0, os.path.join(ROOT_DIR, '..')) | |
| 21 | |
| 22 # In tests/ | |
| 23 import mocks # pylint: disable=W0403 | |
| 24 | |
| 25 from testing_support import auto_stub | |
| 26 | |
| 27 # In root | |
| 28 import buildbot_json | |
| 29 from verification import base | |
| 30 from verification import try_server | |
| 31 | |
| 32 # pylint: disable=W0212 | |
| 33 SUCCESS = buildbot_json.SUCCESS | |
| 34 WARNINGS = buildbot_json.WARNINGS | |
| 35 FAILURE = buildbot_json.FAILURE | |
| 36 SKIPPED = buildbot_json.SKIPPED | |
| 37 EXCEPTION = buildbot_json.EXCEPTION | |
| 38 | |
| 39 | |
| 40 class FakeTryServer(auto_stub.SimpleMock): | |
| 41 """Stateful try server mock. | |
| 42 | |
| 43 Includes calls to send try jobs TryChange() and visible results from HTTP | |
| 44 requests. | |
| 45 """ | |
| 46 def __init__(self, unit_test): | |
| 47 super(FakeTryServer, self).__init__(unit_test) | |
| 48 | |
| 49 # Try server immutable properties. | |
| 50 self.steps = ['update', 'compile', 'test1', 'test2'] | |
| 51 self.server_url = 'http://foo/bar' | |
| 52 | |
| 53 # State of the try server. | |
| 54 self._builds = { 'linux': [], 'mac': [] } | |
| 55 self.pending_builds = { | |
| 56 'linux': [], | |
| 57 'mac': [] | |
| 58 } | |
| 59 # Default mocks. | |
| 60 self.unit_test.mock(urllib, 'urlopen', self._mockurlopen) | |
| 61 self.unit_test.mock(try_server.trychange, 'TryChange', self.TryChangeMock) | |
| 62 | |
| 63 def TryChangeMock(self, cmd, _change, swallow_exception): | |
| 64 """Mocks trychange.py.""" | |
| 65 self.assertEqual(swallow_exception, True) | |
| 66 parser = optparse.OptionParser() | |
| 67 parser.add_option('--bot', action='append') | |
| 68 parser.add_option('--clobber', action='store_true', default=False) | |
| 69 parser.add_option('--email') | |
| 70 parser.add_option('--issue', type='int') | |
| 71 parser.add_option('--name') | |
| 72 parser.add_option('--no_search', action='store_true') | |
| 73 parser.add_option('--patchset', type='int') | |
| 74 parser.add_option('--revision') | |
| 75 parser.add_option('--rietveld_url') | |
| 76 parser.add_option('--user') | |
| 77 options, args = parser.parse_args(cmd) | |
| 78 self.assertEqual(options.email, 'user1@example.com') | |
| 79 self.assertEqual(options.issue, 42) | |
| 80 self.assertEqual(options.no_search, True) | |
| 81 self.assertEqual(options.patchset, 23) | |
| 82 self.assertEqual( | |
| 83 options.rietveld_url, | |
| 84 '%s/download/issue42_23.diff' % self.unit_test.context.rietveld.url) | |
| 85 self.assertEqual(options.user, 'user1') | |
| 86 self.assertEqual(args, ['extra_flags']) | |
| 87 bot = ', '.join(options.bot) | |
| 88 call = 'trychange b={%s} c=%s r=%s' % ( | |
| 89 bot, options.clobber, options.revision) | |
| 90 if options.name != '42-23': | |
| 91 call += ' n=%s' % options.name | |
| 92 self.calls.append(call) | |
| 93 logging.debug(self.calls[-1]) | |
| 94 | |
| 95 def _mockurlopen_internal(self, sub_url): | |
| 96 """Returns data to be encoded before returning.""" | |
| 97 # sub_url is str on python <= 2.6 and unicode for >= 2.7 | |
| 98 self.unit_test.assertTrue(isinstance(sub_url, basestring)) | |
| 99 expected_url = self.server_url + '/json/' | |
| 100 self.unit_test.assertTrue(sub_url.startswith(expected_url)) | |
| 101 self.calls.append(sub_url[len(expected_url):]) | |
| 102 baseurl = '^%s/json/' % re.escape(self.server_url) | |
| 103 match = re.match(baseurl + r'builders/(\w+)/builds/\?(.+)$', sub_url) | |
| 104 if match: | |
| 105 data = {} | |
| 106 for query in match.group(2).split('&'): | |
| 107 m = re.match(r'select=(\w+)', query) | |
| 108 self.unit_test.assertTrue(m, (match.group(2), query)) | |
| 109 build = int(m.group(1)) | |
| 110 data[str(build)] = self._builds[match.group(1)][build] | |
| 111 return data | |
| 112 | |
| 113 match = re.match(baseurl + r'builders/(\w+)/builds/_all$', sub_url) | |
| 114 if match: | |
| 115 # Data is not stored exactly as the try server serves it. | |
| 116 data = {} | |
| 117 for i, build in enumerate(self._builds[match.group(1)]): | |
| 118 data[str(i)] = build | |
| 119 return data | |
| 120 | |
| 121 match = re.match(baseurl + r'builders/\?(.+)$', sub_url) | |
| 122 if match: | |
| 123 data = {} | |
| 124 for query in match.group(1).split('&'): | |
| 125 m = re.match(r'select=(\w+)', query) | |
| 126 self.unit_test.assertTrue(m, (match.group(1), query)) | |
| 127 builder = m.group(1) | |
| 128 data[builder] = { | |
| 129 'cachedBuilds': range(len(self._builds[builder])), | |
| 130 'pendingBuilds': len(self.pending_builds.get(builder, [])), | |
| 131 } | |
| 132 return data | |
| 133 | |
| 134 match = re.match(baseurl + r'builders/(\w+)/pendingBuilds$', sub_url) | |
| 135 if match: | |
| 136 return self.pending_builds[match.group(1)] | |
| 137 | |
| 138 def _mockurlopen(self, sub_url): | |
| 139 """Mocks urllib.urlopen() and JSON encode + StringIO buffers.""" | |
| 140 sub_url = re.match(r'^(.+)[\&\?]filter=1$', sub_url).group(1) | |
| 141 data = self._mockurlopen_internal(sub_url) | |
| 142 self.unit_test.assertNotEquals(None, data, sub_url) | |
| 143 #logging.debug('_mockurlopen(%s) -> %s' % (sub_url, data)) | |
| 144 return StringIO.StringIO(json.dumps(data)) | |
| 145 | |
| 146 def add_build(self, builder, revision, reason, step_results): | |
| 147 """Add a build to a builder.""" | |
| 148 self.assertEqual(len(self.steps), len(step_results)) | |
| 149 assert isinstance(revision, (str, int)) | |
| 150 data = { | |
| 151 'reason': reason or self.unit_test.pending.pending_name(), | |
| 152 'sourceStamp': { | |
| 153 'revision': 'sol@%s' % revision, | |
| 154 'hasPatch': True, | |
| 155 }, | |
| 156 'steps': [], | |
| 157 'blame': ['user1@example.com'], | |
| 158 'number': 0, | |
| 159 'slave': 'foo', | |
| 160 } | |
| 161 result = max(step_results) | |
| 162 if result in (SUCCESS, WARNINGS) and step_results[-1] is None: | |
| 163 result = None | |
| 164 for i, step in enumerate(self.steps): | |
| 165 data['steps'].append({ | |
| 166 'name': step, | |
| 167 'results': [step_results[i]], | |
| 168 }) | |
| 169 data['results'] = [result] | |
| 170 self._builds[builder].append(data) | |
| 171 | |
| 172 def set_build_result(self, builder, result): | |
| 173 """Override the build result for every steps to |result|.""" | |
| 174 for step in self._builds[builder][-1]['steps']: | |
| 175 step['results'] = [result] | |
| 176 self._builds[builder][-1]['results'] = [result] | |
| 177 | |
| 178 | |
| 179 class TryServerSvnTest(mocks.TestCase): | |
| 180 def setUp(self): | |
| 181 super(TryServerSvnTest, self).setUp() | |
| 182 self.email = 'user1@example.com' | |
| 183 self.user = 'user1' | |
| 184 self.timestamp = [1.] | |
| 185 # Mocks http://chromium-status.appspot.com/lkgr | |
| 186 self.lkgr = 123 | |
| 187 self.try_server = FakeTryServer(self) | |
| 188 self.mock(time, 'time', lambda: self.timestamp[-1]) | |
| 189 | |
| 190 try_server.TryRunnerSvn.update_latency = 0 | |
| 191 self.builders_and_tests = { | |
| 192 'linux': ['test1', 'test2'], | |
| 193 'mac': ['test1', 'test2'], | |
| 194 } | |
| 195 self.try_runner = try_server.TryRunnerSvn( | |
| 196 self.context, | |
| 197 self.try_server.server_url, | |
| 198 self.email, | |
| 199 self.builders_and_tests, | |
| 200 ['ignored_step'], | |
| 201 'sol', | |
| 202 ['extra_flags',], | |
| 203 lambda: self.lkgr, | |
| 204 ) | |
| 205 self.pending.revision = 123 | |
| 206 | |
| 207 def tearDown(self): | |
| 208 try: | |
| 209 if not self.has_failed(): | |
| 210 self.try_server.check_calls([]) | |
| 211 finally: | |
| 212 super(TryServerSvnTest, self).tearDown() | |
| 213 | |
| 214 def get_verif(self): | |
| 215 return self.pending.verifications[self.try_runner.name] | |
| 216 | |
| 217 def assertPending( | |
| 218 self, state, nb_jobs, error_message, | |
| 219 linux_build=1, | |
| 220 mac_build=1, | |
| 221 linux_state=None, | |
| 222 mac_state=None, | |
| 223 linux_clobber=False, | |
| 224 mac_clobber=False, | |
| 225 linux_rev=123, | |
| 226 mac_rev=123, | |
| 227 linux_sent=1, | |
| 228 mac_sent=1, | |
| 229 linux_name='42-23', | |
| 230 mac_name='42-23'): | |
| 231 if linux_state is None: | |
| 232 linux_state = state | |
| 233 if mac_state is None: | |
| 234 mac_state = state | |
| 235 self.assertEqual([self.try_runner.name], self.pending.verifications.keys()) | |
| 236 self.assertEqual(error_message, self.get_verif().error_message) | |
| 237 self.assertEqual(nb_jobs, len(self.get_verif().try_jobs)) | |
| 238 self.assertEqual( | |
| 239 len(self.builders_and_tests), len(self.get_verif().try_jobs)) | |
| 240 self.assertEqual(linux_name, self.get_verif().try_jobs[0].name) | |
| 241 self.assertEqual('linux', self.get_verif().try_jobs[0].builder) | |
| 242 self.assertEqual(linux_rev, self.get_verif().try_jobs[0].revision) | |
| 243 self.assertEqual(linux_sent, self.get_verif().try_jobs[0].sent) | |
| 244 self.assertEqual(linux_clobber, self.get_verif().try_jobs[0].clobber) | |
| 245 self.assertEqual(linux_build, self.get_verif().try_jobs[0].build) | |
| 246 self.assertEqual(linux_state, self.get_verif().try_jobs[0].get_state()) | |
| 247 if len(self.builders_and_tests) > 1: | |
| 248 self.assertEqual(mac_name, self.get_verif().try_jobs[1].name) | |
| 249 self.assertEqual('mac', self.get_verif().try_jobs[1].builder) | |
| 250 self.assertEqual(mac_rev, self.get_verif().try_jobs[1].revision) | |
| 251 self.assertEqual(mac_sent, self.get_verif().try_jobs[1].sent) | |
| 252 self.assertEqual(mac_clobber, self.get_verif().try_jobs[1].clobber) | |
| 253 self.assertEqual(mac_build, self.get_verif().try_jobs[1].build) | |
| 254 self.assertEqual(mac_state, self.get_verif().try_jobs[1].get_state()) | |
| 255 self.assertEqual(state, self.get_verif().get_state()) | |
| 256 | |
| 257 def testVoid(self): | |
| 258 self.assertEqual(self.pending.verifications.keys(), []) | |
| 259 | |
| 260 def testVerificationVoid(self): | |
| 261 self.try_runner.verify(self.pending) | |
| 262 self.assertPending(base.PROCESSING, 2, None, linux_build=None, | |
| 263 mac_build=None) | |
| 264 self.try_server.check_calls( | |
| 265 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 266 self.context.status.check_names(['try server'] * 2) | |
| 267 | |
| 268 def testVoidUpdate(self): | |
| 269 self.try_runner.update_status([]) | |
| 270 | |
| 271 def test_steps_quality(self): | |
| 272 self.assertEqual(None, try_server.steps_quality([])) | |
| 273 self.assertEqual(True, try_server.steps_quality([True, None])) | |
| 274 self.assertEqual(False, try_server.steps_quality([True, None, False])) | |
| 275 | |
| 276 def testStepQualityNone(self): | |
| 277 self.try_runner.status.builders['linux'].builds.cache() | |
| 278 self.assertEqual( | |
| 279 (None, 0), | |
| 280 self.try_runner.step_db.revision_quality_builder_steps('linux', 123)) | |
| 281 self.try_server.check_calls(['builders/linux/builds/_all']) | |
| 282 | |
| 283 def testStepQualityGood(self): | |
| 284 self.try_server.add_build( | |
| 285 'linux', 123, None, [SUCCESS, None, None, None]) | |
| 286 self.try_runner.status.builders['linux'].builds.cache() | |
| 287 self.try_server.check_calls(['builders/linux/builds/_all']) | |
| 288 self.assertEqual( | |
| 289 ([True, None, None, None], 1), | |
| 290 self.try_runner.step_db.revision_quality_builder_steps('linux', 123)) | |
| 291 self.try_server.set_build_result('linux', SUCCESS) | |
| 292 self.try_runner.status.builders['linux'].builds.refresh() | |
| 293 self.assertEqual( | |
| 294 ([True] * 4, 1), | |
| 295 self.try_runner.step_db.revision_quality_builder_steps('linux', 123)) | |
| 296 self.try_server.check_calls(['builders/linux/builds/_all']) | |
| 297 | |
| 298 def testStepQualityBad(self): | |
| 299 self.try_server.add_build( | |
| 300 'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, SUCCESS]) | |
| 301 self.try_runner.status.builders['linux'].builds.cache() | |
| 302 # Also test that FakeTryServer.add_build() is implemented correctly. | |
| 303 self.assertEqual( | |
| 304 ([True, True, False, True], 1), | |
| 305 self.try_runner.step_db.revision_quality_builder_steps('linux', 123)) | |
| 306 self.try_server.check_calls(['builders/linux/builds/_all']) | |
| 307 | |
| 308 def testStepQualityBadIncomplete(self): | |
| 309 self.try_server.add_build( | |
| 310 'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, None]) | |
| 311 self.try_runner.status.builders['linux'].builds.cache() | |
| 312 # Also test that FakeTryServer.add_build() is implemented correctly. | |
| 313 self.assertEqual( | |
| 314 ([True, True, False, None], 1), | |
| 315 self.try_runner.step_db.revision_quality_builder_steps('linux', 123)) | |
| 316 self.try_server.check_calls(['builders/linux/builds/_all']) | |
| 317 | |
| 318 def testStepQualityGoodAndBad(self): | |
| 319 self.try_server.add_build( | |
| 320 'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 321 self.try_server.add_build('linux', 123, None, [FAILURE, None, None, None]) | |
| 322 self.try_runner.status.builders['linux'].builds.cache() | |
| 323 self.try_server.check_calls(['builders/linux/builds/_all']) | |
| 324 self.assertEqual( | |
| 325 ([True] * 4, 2), | |
| 326 self.try_runner.step_db.revision_quality_builder_steps('linux', 123)) | |
| 327 | |
| 328 def testQualityAutomatic(self): | |
| 329 self.try_runner.verify(self.pending) | |
| 330 self.assertEqual( | |
| 331 (None, 0), | |
| 332 self.try_runner.step_db.revision_quality_builder_steps( | |
| 333 'linux', 123)) | |
| 334 self.try_server.add_build( | |
| 335 'linux', 123, 'georges tried stuff', | |
| 336 [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 337 self.try_runner.update_status([self.pending]) | |
| 338 self.assertEqual( | |
| 339 ([True] * 4, 1), | |
| 340 self.try_runner.step_db.revision_quality_builder_steps( | |
| 341 'linux', 123)) | |
| 342 self.try_server.check_calls( | |
| 343 [ 'trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123', | |
| 344 'builders/?select=linux&select=mac', | |
| 345 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 346 self.context.status.check_names(['try server'] * 2) | |
| 347 | |
| 348 def testQualityManual(self): | |
| 349 self.try_server.add_build( | |
| 350 'linux', 123, 'georges tried stuff', | |
| 351 [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 352 self.try_runner.status.builders['linux'].builds.cache() | |
| 353 self.assertEqual( | |
| 354 ([True] * 4, 1), | |
| 355 self.try_runner.step_db.revision_quality_builder_steps( | |
| 356 'linux', 123)) | |
| 357 self.try_server.check_calls(['builders/linux/builds/_all']) | |
| 358 | |
| 359 def _simple(self, status_linux, status_mac=None, error_msg=None): | |
| 360 """status_linux affects test1, status_mac affects test2.""" | |
| 361 def is_failure(status): | |
| 362 return status in (FAILURE, EXCEPTION) | |
| 363 | |
| 364 self.assertEqual( | |
| 365 bool(is_failure(status_linux) or is_failure(status_mac)), | |
| 366 bool(error_msg)) | |
| 367 if status_mac is None: | |
| 368 status_mac = status_linux | |
| 369 self.lkgr = 12 | |
| 370 self.try_server.add_build( | |
| 371 'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 372 self.try_server.add_build( | |
| 373 'mac', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 374 | |
| 375 self.try_runner.verify(self.pending) | |
| 376 self.try_server.check_calls( | |
| 377 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 378 self.try_server.add_build( | |
| 379 'linux', 123, None, [SUCCESS, SUCCESS, status_linux, SUCCESS]) | |
| 380 self.try_server.add_build( | |
| 381 'mac', 123, None, [SUCCESS, SUCCESS, SUCCESS, status_mac]) | |
| 382 self.try_runner.update_status([self.pending]) | |
| 383 | |
| 384 if is_failure(status_linux): | |
| 385 self.assertEqual(123, self.try_runner.get_lkgr('linux')) | |
| 386 self.assertEqual( | |
| 387 123, self.try_runner.step_db.last_good_revision_builder('linux')) | |
| 388 else: | |
| 389 self.assertEqual(123, self.try_runner.get_lkgr('linux')) | |
| 390 self.assertEqual( | |
| 391 123, self.try_runner.step_db.last_good_revision_builder('linux')) | |
| 392 if is_failure(status_mac): | |
| 393 self.assertEqual(123, self.try_runner.get_lkgr('mac')) | |
| 394 self.assertEqual( | |
| 395 123, self.try_runner.step_db.last_good_revision_builder('mac')) | |
| 396 else: | |
| 397 self.assertEqual(123, self.try_runner.get_lkgr('mac')) | |
| 398 self.assertEqual( | |
| 399 123, self.try_runner.step_db.last_good_revision_builder('mac')) | |
| 400 | |
| 401 if error_msg: | |
| 402 # Can't test failure without testing automatic retry mechanism. | |
| 403 expected = ( | |
| 404 [ 'builders/?select=linux&select=mac', | |
| 405 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 406 | |
| 407 if is_failure(status_linux): | |
| 408 expected.append( | |
| 409 'trychange b={linux:test1} c=False r=sol@123 n=42-23 (retry)') | |
| 410 linux_build = None | |
| 411 linux_state = base.PROCESSING | |
| 412 linux_name = '42-23 (retry)' | |
| 413 else: | |
| 414 linux_build = 1 | |
| 415 linux_state = base.SUCCEEDED | |
| 416 linux_name = '42-23' | |
| 417 | |
| 418 if is_failure(status_mac): | |
| 419 expected.append( | |
| 420 'trychange b={mac:test2} c=False r=sol@123 n=42-23 (retry)') | |
| 421 mac_build = None | |
| 422 mac_state = base.PROCESSING | |
| 423 mac_name = '42-23 (retry)' | |
| 424 else: | |
| 425 mac_build = 1 | |
| 426 mac_state = base.SUCCEEDED | |
| 427 mac_name = '42-23' | |
| 428 | |
| 429 self.assertPending( | |
| 430 base.PROCESSING, 2, None, linux_build=linux_build, | |
| 431 mac_build=mac_build, linux_state=linux_state, mac_state=mac_state, | |
| 432 linux_name=linux_name, mac_name=mac_name) | |
| 433 self.try_server.check_calls(expected) | |
| 434 | |
| 435 if is_failure(status_linux): | |
| 436 self.try_server.add_build( | |
| 437 'linux', 123, linux_name, [SUCCESS, SUCCESS, status_linux, SUCCESS]) | |
| 438 if is_failure(status_mac): | |
| 439 self.try_server.add_build( | |
| 440 'mac', 123, mac_name, [SUCCESS, SUCCESS, SUCCESS, status_mac]) | |
| 441 | |
| 442 self.try_runner.update_status([self.pending]) | |
| 443 if is_failure(status_linux): | |
| 444 linux_state = base.FAILED | |
| 445 linux_build = 2 | |
| 446 else: | |
| 447 linux_build = 1 | |
| 448 if is_failure(status_mac): | |
| 449 mac_state = base.FAILED | |
| 450 mac_build = 2 | |
| 451 else: | |
| 452 mac_build = 1 | |
| 453 self.assertPending( | |
| 454 base.FAILED, 2, error_msg, | |
| 455 linux_build=linux_build, mac_build=mac_build, | |
| 456 linux_state=linux_state, mac_state=mac_state, | |
| 457 linux_name=linux_name, mac_name=mac_name) | |
| 458 | |
| 459 if is_failure(status_linux) and is_failure(status_mac): | |
| 460 self.try_server.check_calls( | |
| 461 [ 'builders/?select=linux&select=mac', | |
| 462 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 463 self.context.checkout.check_calls( | |
| 464 [ 'prepare(123)', | |
| 465 'apply_patch(%r)' % self.context.rietveld.patchsets[-2], | |
| 466 'prepare(123)', | |
| 467 'apply_patch(%r)' % self.context.rietveld.patchsets[-1]]) | |
| 468 elif is_failure(status_linux): | |
| 469 self.try_server.check_calls( | |
| 470 ['builders/?select=linux', 'builders/linux/builds/_all']) | |
| 471 self.context.checkout.check_calls( | |
| 472 [ 'prepare(123)', | |
| 473 'apply_patch(%r)' % self.context.rietveld.patchsets[-1]]) | |
| 474 else: | |
| 475 self.try_server.check_calls( | |
| 476 ['builders/?select=mac', 'builders/mac/builds/_all']) | |
| 477 self.context.checkout.check_calls( | |
| 478 [ 'prepare(123)', | |
| 479 'apply_patch(%r)' % self.context.rietveld.patchsets[-1]]) | |
| 480 else: | |
| 481 self.assertPending(base.SUCCEEDED, 2, None) | |
| 482 self.try_server.check_calls( | |
| 483 [ 'builders/?select=linux&select=mac', | |
| 484 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 485 count = 6 + 3 * ( | |
| 486 int(is_failure(status_linux)) + int(is_failure(status_mac))) | |
| 487 self.context.status.check_names(['try server'] * count) | |
| 488 | |
| 489 def testImmediateSuccess(self): | |
| 490 self._simple(SUCCESS) | |
| 491 | |
| 492 def testImmediateWarnings(self): | |
| 493 self._simple(WARNINGS) | |
| 494 | |
| 495 def testImmediateSkipped(self): | |
| 496 self._simple(SKIPPED) | |
| 497 | |
| 498 def second_fail_msg( | |
| 499 self, clname, step2, step1, builder, number, is_clobber=False): | |
| 500 extra = '' | |
| 501 if is_clobber: | |
| 502 extra = ' (clobber build)' | |
| 503 return ( | |
| 504 u'Try job failure for %s on %s for step "%s"%s.\n' | |
| 505 u'It\'s a second try, previously, step "%s" failed.\n' | |
| 506 u'%s/buildstatus?builder=%s&number=%s\n') % ( | |
| 507 clname, builder, step2, extra, step1, self.try_server.server_url, | |
| 508 builder, number) | |
| 509 def testImmediateFailureLinux(self): | |
| 510 self._simple( | |
| 511 FAILURE, SUCCESS, | |
| 512 self.second_fail_msg('42-23 (retry)', 'test1', 'test1', 'linux', 2)) | |
| 513 | |
| 514 def testImmediateFailureMac(self): | |
| 515 self._simple( | |
| 516 SUCCESS, FAILURE, | |
| 517 self.second_fail_msg('42-23 (retry)', 'test2', 'test2', 'mac', 2)) | |
| 518 | |
| 519 def testImmediateDoubleFailure(self): | |
| 520 self._simple( | |
| 521 FAILURE, FAILURE, | |
| 522 self.second_fail_msg('42-23 (retry)', 'test2', 'test2', 'mac', 2)) | |
| 523 | |
| 524 def testImmediateException(self): | |
| 525 self._simple( | |
| 526 SUCCESS, EXCEPTION, | |
| 527 self.second_fail_msg('42-23 (retry)', 'test2', 'test2', 'mac', 2)) | |
| 528 | |
| 529 def testSuccess(self): | |
| 530 self.lkgr = 2 | |
| 531 # Normal workflow with incremental success. | |
| 532 self.try_runner.verify(self.pending) | |
| 533 self.try_server.check_calls( | |
| 534 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 535 | |
| 536 self.try_runner.update_status([self.pending]) | |
| 537 self.assertPending( | |
| 538 base.PROCESSING, 2, None, linux_build=None, mac_build=None) | |
| 539 self.try_server.check_calls( | |
| 540 ['builders/?select=linux&select=mac', | |
| 541 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 542 | |
| 543 self.try_server.add_build( | |
| 544 'linux', 123, None, [SUCCESS, None, None, None]) | |
| 545 self.try_runner.update_status([self.pending]) | |
| 546 self.assertPending(base.PROCESSING, 2, None, linux_build=0, mac_build=None) | |
| 547 self.try_server.check_calls( | |
| 548 ['builders/?select=linux&select=mac', | |
| 549 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 550 | |
| 551 self.try_server.add_build( | |
| 552 'mac', 123, None, [SUCCESS, None, None, None]) | |
| 553 self.try_runner.update_status([self.pending]) | |
| 554 self.assertPending(base.PROCESSING, 2, None, linux_build=0, mac_build=0) | |
| 555 self.try_server.check_calls( | |
| 556 ['builders/?select=mac', | |
| 557 'builders/mac/builds/_all', 'builders/linux/builds/?select=0']) | |
| 558 | |
| 559 # This one will be cached since it's now immutable. | |
| 560 self.try_server.set_build_result('mac', SUCCESS) | |
| 561 self.try_runner.update_status([self.pending]) | |
| 562 self.assertPending( | |
| 563 base.PROCESSING, 2, None, linux_build=0, mac_build=0, | |
| 564 mac_state=base.SUCCEEDED) | |
| 565 self.try_server.check_calls( | |
| 566 ['builders/linux/builds/?select=0', 'builders/mac/builds/?select=0']) | |
| 567 | |
| 568 self.try_server.set_build_result('linux', SUCCESS) | |
| 569 self.try_runner.update_status([self.pending]) | |
| 570 self.assertPending(base.SUCCEEDED, 2, None, linux_build=0, mac_build=0) | |
| 571 self.assertEqual( | |
| 572 123, self.try_runner.step_db.last_good_revision_builder('linux')) | |
| 573 self.assertEqual( | |
| 574 123, self.try_runner.step_db.last_good_revision_builder('mac')) | |
| 575 self.assertEqual( | |
| 576 ([True] * 4, 1), | |
| 577 self.try_runner.step_db.revision_quality_builder_steps( | |
| 578 'linux', 123)) | |
| 579 self.assertEqual( | |
| 580 ([True] * 4, 1), | |
| 581 self.try_runner.step_db.revision_quality_builder_steps( | |
| 582 'mac', 123)) | |
| 583 self.try_server.check_calls(['builders/linux/builds/?select=0']) | |
| 584 self.context.status.check_names(['try server'] * 6) | |
| 585 | |
| 586 def testIgnorePreviousJobs(self): | |
| 587 self.try_runner.verify(self.pending) | |
| 588 self.try_server.check_calls( | |
| 589 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 590 | |
| 591 self.try_runner.update_status([self.pending]) | |
| 592 self.try_server.check_calls( | |
| 593 [ 'builders/?select=linux&select=mac', | |
| 594 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 595 | |
| 596 self.try_server.add_build('linux', 12, None, [None, None, None, None]) | |
| 597 self.try_server.add_build('mac', 12, None, [None, None, None, None]) | |
| 598 self.try_runner.update_status([self.pending]) | |
| 599 self.assertPending( | |
| 600 base.PROCESSING, 2, None, linux_build=None, | |
| 601 mac_build=None) | |
| 602 self.try_server.check_calls( | |
| 603 ['builders/?select=linux&select=mac', | |
| 604 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 605 | |
| 606 self.try_server.add_build( | |
| 607 'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 608 self.try_server.add_build( | |
| 609 'mac', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 610 self.try_runner.update_status([self.pending]) | |
| 611 self.assertPending(base.SUCCEEDED, 2, None) | |
| 612 self.assertEqual( | |
| 613 123, self.try_runner.step_db.last_good_revision_builder('linux')) | |
| 614 self.assertEqual( | |
| 615 123, self.try_runner.step_db.last_good_revision_builder('mac')) | |
| 616 self.assertEqual( | |
| 617 ([True] * 4, 1), | |
| 618 self.try_runner.step_db.revision_quality_builder_steps( | |
| 619 'linux', 123)) | |
| 620 self.assertEqual( | |
| 621 ([True] * 4, 1), | |
| 622 self.try_runner.step_db.revision_quality_builder_steps( | |
| 623 'mac', 123)) | |
| 624 self.try_server.check_calls( | |
| 625 ['builders/?select=linux&select=mac', | |
| 626 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 627 self.context.status.check_names(['try server'] * 6) | |
| 628 | |
| 629 def testNames(self): | |
| 630 job = try_server.TryJob( | |
| 631 builder='builder', revision=123, tests=['test1'], clobber=False) | |
| 632 self.assertEqual(None, job.name) | |
| 633 | |
| 634 def testLostJob(self): | |
| 635 # Test that a job is automatically retried if it was never started up. It | |
| 636 # does happen. | |
| 637 self.try_runner.verify(self.pending) | |
| 638 self.try_server.check_calls( | |
| 639 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 640 | |
| 641 # Keep a copy of the try jobs to compare later. | |
| 642 self.try_runner.update_status([self.pending]) | |
| 643 self.try_server.check_calls( | |
| 644 [ 'builders/?select=linux&select=mac', | |
| 645 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 646 self.assertPending( | |
| 647 base.PROCESSING, 2, None, mac_sent=self.timestamp[-1], | |
| 648 linux_build=None, mac_build=None) | |
| 649 | |
| 650 # lost_try_job_delay + 2 seconds later. | |
| 651 # linux is pending, mac is lost. | |
| 652 self.try_server.pending_builds['linux'] = [ | |
| 653 { | |
| 654 'reason': '42-23', | |
| 655 } | |
| 656 ] | |
| 657 self.timestamp.append(self.try_runner.lost_try_job_delay + 2) | |
| 658 self.try_runner.update_status([self.pending]) | |
| 659 self.assertPending( | |
| 660 base.PROCESSING, 2, None, mac_sent=self.timestamp[-1], | |
| 661 linux_build=None, mac_build=None, | |
| 662 mac_name='42-23 (previous was lost)') | |
| 663 self.try_server.check_calls( | |
| 664 # Look if there is pending build on each builder. | |
| 665 [ 'builders/?select=linux&select=mac', | |
| 666 'builders/linux/builds/_all', 'builders/mac/builds/_all', | |
| 667 'builders/linux/pendingBuilds', | |
| 668 # Retry only mac. | |
| 669 'trychange b={mac:test1,test2} c=False r=sol@123 n=42-23 (previous ' | |
| 670 'was lost)']) | |
| 671 | |
| 672 # linux job was completed. | |
| 673 self.try_server.pending_builds['linux'] = [] | |
| 674 self.try_server.add_build( | |
| 675 'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 676 self.try_runner.update_status([self.pending]) | |
| 677 self.assertPending( | |
| 678 base.PROCESSING, 2, None, mac_sent=self.timestamp[1], | |
| 679 linux_build=0, mac_build=None, | |
| 680 linux_state=base.SUCCEEDED, | |
| 681 mac_name='42-23 (previous was lost)') | |
| 682 self.try_server.check_calls( | |
| 683 [ 'builders/?select=linux&select=mac', | |
| 684 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 685 | |
| 686 # 2 * (lost_try_job_delay + 2) seconds later, mac job started and completed. | |
| 687 self.timestamp.append(2 * (self.try_runner.lost_try_job_delay + 2)) | |
| 688 self.try_server.add_build( | |
| 689 'mac', 123, '42-23 (previous was lost)', | |
| 690 [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 691 self.try_runner.update_status([self.pending]) | |
| 692 self.assertPending( | |
| 693 base.SUCCEEDED, 2, None, mac_sent=self.timestamp[1], | |
| 694 linux_build=0, mac_build=0, | |
| 695 mac_name='42-23 (previous was lost)') | |
| 696 self.try_server.check_calls( | |
| 697 ['builders/?select=mac', 'builders/mac/builds/_all']) | |
| 698 | |
| 699 self.try_runner.update_status([self.pending]) | |
| 700 self.context.checkout.check_calls( | |
| 701 [ 'prepare(123)', | |
| 702 'apply_patch(%r)' % self.context.rietveld.patchsets[-1]]) | |
| 703 self.context.status.check_names(['try server'] * 7) | |
| 704 | |
| 705 def testFailedStepRetryLkgr(self): | |
| 706 self.try_runner.verify(self.pending) | |
| 707 self.try_server.check_calls( | |
| 708 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 709 self.try_server.add_build( | |
| 710 'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, SUCCESS]) | |
| 711 | |
| 712 self.lkgr = 122 | |
| 713 self.try_runner.update_status([self.pending]) | |
| 714 self.try_server.check_calls( | |
| 715 [ 'builders/?select=linux&select=mac', | |
| 716 'builders/linux/builds/_all', 'builders/mac/builds/_all', | |
| 717 # Only the failed test is retried, on lkgr revision. | |
| 718 'trychange b={linux:test1} c=False r=sol@122 n=42-23 (retry)']) | |
| 719 self.assertEqual(['test1'], self.get_verif().try_jobs[0].failed_steps) | |
| 720 self.context.checkout.check_calls( | |
| 721 [ 'prepare(122)', | |
| 722 'apply_patch(%r)' % self.context.rietveld.patchsets[-1]]) | |
| 723 self.context.status.check_names(['try server'] * 5) | |
| 724 | |
| 725 def testFailedUpdate(self): | |
| 726 # It must not retry a failed update. | |
| 727 # Add succeededing builds, this sets quality to True, which disable retry | |
| 728 # mechanism. | |
| 729 self.try_server.add_build( | |
| 730 'linux', 123, 'georges tried stuff', | |
| 731 [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 732 self.try_server.add_build( | |
| 733 'mac', 123, 'georges tried stuff', [SUCCESS, SUCCESS, SUCCESS, SUCCESS]) | |
| 734 self.lkgr = 123 | |
| 735 | |
| 736 self.try_runner.verify(self.pending) | |
| 737 self.try_server.check_calls( | |
| 738 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 739 self.try_server.add_build( | |
| 740 'linux', 123, None, [FAILURE, None, None, None]) | |
| 741 self.try_runner.update_status([self.pending]) | |
| 742 self.try_server.check_calls( | |
| 743 [ 'builders/?select=linux&select=mac', | |
| 744 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 745 self.assertEqual('linux', self.get_verif().try_jobs[0].builder) | |
| 746 self.assertEqual(['update'], self.get_verif().try_jobs[0].failed_steps) | |
| 747 self.assertPending( | |
| 748 base.FAILED, 2, | |
| 749 (u'Try job failure for 42-23 on linux for step ' | |
| 750 u'"update".\n%s/buildstatus?builder=linux&number=1\n\n' | |
| 751 u'Step "update" is always a major failure.\n' | |
| 752 u'Look at the try server FAQ for more details.') % | |
| 753 self.try_server.server_url, | |
| 754 mac_build=None, | |
| 755 mac_state=base.PROCESSING) | |
| 756 self.context.status.check_names(['try server'] * 4) | |
| 757 | |
| 758 def testFailedCompileRetryClobber(self): | |
| 759 # It must retry once a non-clobber compile. | |
| 760 self.try_runner.verify(self.pending) | |
| 761 self.try_server.check_calls( | |
| 762 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 763 self.try_server.add_build( | |
| 764 'linux', 123, None, [SUCCESS, FAILURE, None, None]) | |
| 765 self.lkgr = 122 | |
| 766 self.try_runner.update_status([self.pending]) | |
| 767 self.context.checkout.check_calls( | |
| 768 [ 'prepare(122)', | |
| 769 'apply_patch(%r)' % self.context.rietveld.patchsets[-1]]) | |
| 770 self.try_server.check_calls( | |
| 771 [ 'builders/?select=linux&select=mac', | |
| 772 'builders/linux/builds/_all', 'builders/mac/builds/_all', | |
| 773 # Retries at lkgr. | |
| 774 'trychange b={linux:test1,test2} c=True r=sol@122 n=42-23 (retry)']) | |
| 775 self.assertEqual(['compile'], self.get_verif().try_jobs[0].failed_steps) | |
| 776 self.assertPending( | |
| 777 base.PROCESSING, 2, None, linux_rev=122, | |
| 778 linux_clobber=True, linux_build=None, mac_build=None, | |
| 779 linux_name='42-23 (retry)') | |
| 780 | |
| 781 self.try_server.add_build( | |
| 782 'linux', 122, '42-23 (retry)', [SUCCESS, FAILURE, None, None]) | |
| 783 self.try_runner.update_status([self.pending]) | |
| 784 self.try_server.check_calls( | |
| 785 [ 'builders/?select=linux&select=mac', | |
| 786 'builders/linux/builds/_all', 'builders/mac/builds/_all']) | |
| 787 self.assertEqual(['compile'], self.get_verif().try_jobs[0].failed_steps) | |
| 788 self.assertPending( | |
| 789 base.FAILED, 2, | |
| 790 self.second_fail_msg('42-23 (retry)', 'compile', 'compile', 'linux', 1, | |
| 791 True), | |
| 792 linux_rev=122, | |
| 793 linux_clobber=True, | |
| 794 mac_build=None, | |
| 795 mac_state=base.PROCESSING, | |
| 796 linux_name='42-23 (retry)') | |
| 797 self.context.status.check_names(['try server'] * 7) | |
| 798 | |
| 799 def testTooManyRetries(self): | |
| 800 self.try_runner.verify(self.pending) | |
| 801 self.try_server.check_calls( | |
| 802 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 803 job = self.pending.verifications[self.try_runner.name].try_jobs[0] | |
| 804 self.try_runner._send_jobs( | |
| 805 self.pending, [job], False, {job.builder:job.tests}, 'foo') | |
| 806 self.try_server.check_calls( | |
| 807 [ 'trychange b={linux:test1,test2} c=False r=sol@123 n=foo']) | |
| 808 job = self.pending.verifications[self.try_runner.name].try_jobs[0] | |
| 809 self.try_runner._send_jobs( | |
| 810 self.pending, [job], False, {job.builder:job.tests}, 'foo') | |
| 811 self.try_server.check_calls( | |
| 812 [ 'trychange b={linux:test1,test2} c=False r=sol@123 n=foo']) | |
| 813 job = self.pending.verifications[self.try_runner.name].try_jobs[0] | |
| 814 self.try_runner._send_jobs( | |
| 815 self.pending, [job], False, {job.builder:job.tests}, 'foo') | |
| 816 self.try_server.check_calls( | |
| 817 [ 'trychange b={linux:test1,test2} c=False r=sol@123 n=foo']) | |
| 818 job = self.pending.verifications[self.try_runner.name].try_jobs[0] | |
| 819 try: | |
| 820 self.try_runner._send_jobs( | |
| 821 self.pending, [job], False, {job.builder:job.tests}, 'foo') | |
| 822 self.fail() | |
| 823 except base.DiscardPending: | |
| 824 pass | |
| 825 self.context.status.check_names(['try server'] * 5) | |
| 826 | |
| 827 def testNoTry(self): | |
| 828 self.pending.description += '\nNOTRY=true' | |
| 829 self.try_runner.verify(self.pending) | |
| 830 self.assertEqual( | |
| 831 base.SUCCEEDED, | |
| 832 self.pending.verifications[self.try_runner.name].get_state()) | |
| 833 | |
| 834 def testNoTryWrong(self): | |
| 835 self.pending.description += '\nNOTRY=true2' | |
| 836 self.try_runner.verify(self.pending) | |
| 837 self.try_server.check_calls( | |
| 838 ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123']) | |
| 839 self.context.status.check_names(['try server'] * 2) | |
| 840 self.assertEqual( | |
| 841 base.PROCESSING, | |
| 842 self.pending.verifications[self.try_runner.name].get_state()) | |
| 843 | |
| 844 def testSuccessIgnoredFailure(self): | |
| 845 # Simplify testing code by removing mac. | |
| 846 del self.try_runner.builders_and_tests['mac'] | |
| 847 # Add fake failing ignored step. | |
| 848 self.try_server.steps = [ | |
| 849 'update', 'compile', 'ignored_step', 'test1', 'test2'] | |
| 850 | |
| 851 self.try_runner.verify(self.pending) | |
| 852 self.try_server.check_calls( | |
| 853 ['trychange b={linux:test1,test2} c=False r=sol@123']) | |
| 854 | |
| 855 self.try_runner.update_status([self.pending]) | |
| 856 self.assertPending( | |
| 857 base.PROCESSING, 1, None, linux_build=None, mac_build=None) | |
| 858 self.try_server.check_calls( | |
| 859 ['builders/?select=linux', 'builders/linux/builds/_all']) | |
| 860 | |
| 861 self.try_server.add_build( | |
| 862 'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, SUCCESS, SUCCESS]) | |
| 863 self.try_runner.update_status([self.pending]) | |
| 864 self.assertPending(base.SUCCEEDED, 1, None, linux_build=0, mac_build=None) | |
| 865 self.try_server.check_calls( | |
| 866 ['builders/?select=linux', 'builders/linux/builds/_all']) | |
| 867 | |
| 868 self.try_server.set_build_result('linux', SUCCESS) | |
| 869 self.try_runner.update_status([self.pending]) | |
| 870 self.assertPending(base.SUCCEEDED, 1, None, linux_build=0, mac_build=0) | |
| 871 # TODO(maruel): Fix, since StepDb doesn't know about ignored steps. | |
| 872 self.assertEqual( | |
| 873 None, self.try_runner.step_db.last_good_revision_builder('linux')) | |
| 874 self.assertEqual( | |
| 875 ([True, True, False, True, True], 1), | |
| 876 self.try_runner.step_db.revision_quality_builder_steps( | |
| 877 'linux', 123)) | |
| 878 self.context.status.check_names(['try server'] * 3) | |
| 879 | |
| 880 | |
| 881 if __name__ == '__main__': | |
| 882 logging.basicConfig( | |
| 883 level=[logging.WARNING, logging.INFO, logging.DEBUG][ | |
| 884 min(sys.argv.count('-v'), 2)], | |
| 885 format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s') | |
| 886 unittest.main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 135363007:
Delete public commit queue to avoid confusion after move to internal repo (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/