| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 | |
| 6 """Miscellaneous utilities needed by the Skia buildbot master.""" | |
| 7 | |
| 8 | |
| 9 import difflib | |
| 10 import httplib2 | |
| 11 import json | |
| 12 import os | |
| 13 import re | |
| 14 | |
| 15 # requires Google APIs client library for Python; see | |
| 16 # https://code.google.com/p/google-api-python-client/wiki/Installation | |
| 17 from apiclient.discovery import build | |
| 18 from buildbot.scheduler import Dependent | |
| 19 from buildbot.scheduler import Scheduler | |
| 20 from buildbot.schedulers import timed | |
| 21 from buildbot.schedulers.filter import ChangeFilter | |
| 22 from config_private import TRY_SVN_BASEURL | |
| 23 from master import try_job_svn | |
| 24 from master import try_job_rietveld | |
| 25 from master.builders_pools import BuildersPools | |
| 26 from oauth2client.client import SignedJwtAssertionCredentials | |
| 27 | |
| 28 import builder_name_schema | |
| 29 import config_private | |
| 30 import os | |
| 31 import skia_vars | |
| 32 import subprocess | |
| 33 | |
| 34 | |
| 35 GATEKEEPER_NAME = 'GateKeeper' | |
| 36 | |
| 37 TRY_SCHEDULER_SVN = 'skia_try_svn' | |
| 38 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' | |
| 39 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] | |
| 40 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) | |
| 41 | |
| 42 | |
| 43 def GetListFromEnvVar(name, splitstring=','): | |
| 44 """ Returns contents of an environment variable, as a list. | |
| 45 | |
| 46 If the environment variable is unset or set to empty-string, this returns | |
| 47 an empty list. | |
| 48 | |
| 49 name: string; name of the environment variable to read | |
| 50 splitstring: string with which to split the env var into list items | |
| 51 """ | |
| 52 unsplit = os.environ.get(name, None) | |
| 53 if unsplit: | |
| 54 return unsplit.split(',') | |
| 55 else: | |
| 56 return [] | |
| 57 | |
| 58 | |
| 59 def StringDiff(expected, actual): | |
| 60 """ Returns the diff between two multiline strings, as a multiline string.""" | |
| 61 return ''.join(difflib.unified_diff(expected.splitlines(1), | |
| 62 actual.splitlines(1))) | |
| 63 | |
| 64 | |
| 65 def ToString(obj): | |
| 66 """ Returns a string representation of the given object. This differs from the | |
| 67 built-in string function in that it does not give memory locations. | |
| 68 | |
| 69 obj: the object to print. | |
| 70 """ | |
| 71 def sanitize(obj): | |
| 72 if isinstance(obj, list) or isinstance(obj, tuple): | |
| 73 return [sanitize(sub_obj) for sub_obj in obj] | |
| 74 elif isinstance(obj, dict): | |
| 75 rv = {} | |
| 76 for k, v in obj.iteritems(): | |
| 77 rv[k] = sanitize(v) | |
| 78 return rv | |
| 79 elif isinstance(obj, str) or obj is None: | |
| 80 return obj | |
| 81 else: | |
| 82 return '<Object>' | |
| 83 return json.dumps(sanitize(obj), indent=4, sort_keys=True) | |
| 84 | |
| 85 | |
| 86 def FixGitSvnEmail(addr): | |
| 87 """ Git-svn tacks a git-svn-id onto email addresses. This function removes it. | |
| 88 | |
| 89 For example, "skia.buildbots@gmail.com@2bbb7eff-a529-9590-31e7-b0007b416f81" | |
| 90 becomes, "skia.buildbots@gmail.com". Addresses containing a single '@' will be | |
| 91 unchanged. | |
| 92 """ | |
| 93 return '@'.join(addr.split('@')[:2]) | |
| 94 | |
| 95 | |
| 96 class SkiaChangeFilter(ChangeFilter): | |
| 97 """Skia specific subclass of ChangeFilter.""" | |
| 98 | |
| 99 def __init__(self, builders, **kwargs): | |
| 100 self._builders = builders | |
| 101 ChangeFilter.__init__(self, **kwargs) | |
| 102 | |
| 103 def filter_change(self, change): | |
| 104 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. | |
| 105 | |
| 106 The code has been copied from | |
| 107 http://buildbot.net/buildbot/docs/0.8.3/reference/buildbot.schedulers.filter
-pysrc.html#ChangeFilter | |
| 108 with one change: We pass a sequence of builders to the filter function. | |
| 109 """ | |
| 110 if self.filter_fn is not None and not self.filter_fn(change, | |
| 111 self._builders): | |
| 112 return False | |
| 113 for (filt_list, filt_re, filt_fn, chg_attr) in self.checks: | |
| 114 chg_val = getattr(change, chg_attr, '') | |
| 115 if filt_list is not None and chg_val not in filt_list: | |
| 116 return False | |
| 117 if filt_re is not None and ( | |
| 118 chg_val is None or not filt_re.match(chg_val)): | |
| 119 return False | |
| 120 if filt_fn is not None and not filt_fn(chg_val): | |
| 121 return False | |
| 122 return True | |
| 123 | |
| 124 | |
| 125 def _AssertValidString(var, varName='[unknown]'): | |
| 126 """Raises an exception if a var is not a valid string. | |
| 127 | |
| 128 A string is considered valid if it is not None, is not the empty string and is | |
| 129 not just whitespace. | |
| 130 | |
| 131 Args: | |
| 132 var: the variable to validate | |
| 133 varName: name of the variable, for error reporting | |
| 134 """ | |
| 135 if not isinstance(var, str): | |
| 136 raise Exception('variable "%s" is not a string' % varName) | |
| 137 if not var: | |
| 138 raise Exception('variable "%s" is empty' % varName) | |
| 139 if var.isspace(): | |
| 140 raise Exception('variable "%s" is whitespace' % varName) | |
| 141 | |
| 142 | |
| 143 def _AssertValidStringList(var, varName='[unknown]'): | |
| 144 """Raises an exception if var is not a list of valid strings. | |
| 145 | |
| 146 A list is considered valid if it is either empty or if it contains at | |
| 147 least one item and each item it contains is also a valid string. | |
| 148 | |
| 149 Args: | |
| 150 var: the variable to validate | |
| 151 varName: name of the variable, for error reporting | |
| 152 """ | |
| 153 if not isinstance(var, list): | |
| 154 raise Exception('variable "%s" is not a list' % varName) | |
| 155 for index, item in zip(range(len(var)), var): | |
| 156 _AssertValidString(item, '%s[%d]' % (varName, index)) | |
| 157 | |
| 158 | |
| 159 def FileBug(summary, description, owner=None, ccs=None, labels=None): | |
| 160 """Files a bug to the Skia issue tracker. | |
| 161 | |
| 162 Args: | |
| 163 summary: a single-line string to use as the issue summary | |
| 164 description: a multiline string to use as the issue description | |
| 165 owner: email address of the issue owner (as a string), or None if unknown | |
| 166 ccs: email addresses (list of strings) to CC on the bug | |
| 167 labels: labels (list of strings) to apply to the bug | |
| 168 | |
| 169 Returns: | |
| 170 A representation of the issue tracker issue that was filed or raises an | |
| 171 exception if there was a problem. | |
| 172 """ | |
| 173 project_id = 'skia' # This is the project name: skia | |
| 174 key_file = 'key.p12' # Key file from the API console, renamed to key.p12 | |
| 175 service_acct = ('352371350305-b3u8jq5sotdh964othi9ntg9d0pelu77' | |
| 176 '@developer.gserviceaccount.com') # Created with the key | |
| 177 result = {} | |
| 178 if not ccs: | |
| 179 ccs = [] | |
| 180 if not labels: | |
| 181 labels = [] | |
| 182 | |
| 183 if owner is not None: # owner can be None | |
| 184 _AssertValidString(owner, 'owner') | |
| 185 _AssertValidString(summary, 'summary') | |
| 186 _AssertValidString(description, 'description') | |
| 187 _AssertValidStringList(ccs, 'ccs') | |
| 188 _AssertValidStringList(labels, 'labels') | |
| 189 | |
| 190 f = file(key_file, 'rb') | |
| 191 key = f.read() | |
| 192 f.close() | |
| 193 | |
| 194 # Create an httplib2.Http object to handle the HTTP requests and authorize | |
| 195 # it with the credentials. | |
| 196 credentials = SignedJwtAssertionCredentials( | |
| 197 service_acct, | |
| 198 key, | |
| 199 scope='https://www.googleapis.com/auth/projecthosting') | |
| 200 http = httplib2.Http() | |
| 201 http = credentials.authorize(http) | |
| 202 | |
| 203 service = build("projecthosting", "v2", http=http) | |
| 204 | |
| 205 # Insert a new issue into the project. | |
| 206 body = { | |
| 207 'summary': summary, | |
| 208 'description': description | |
| 209 } | |
| 210 | |
| 211 insertparams = { | |
| 212 'projectId': project_id, | |
| 213 'sendEmail': 'true' | |
| 214 } | |
| 215 | |
| 216 if owner is not None: | |
| 217 owner_value = { | |
| 218 'name': owner | |
| 219 } | |
| 220 body['owner'] = owner_value | |
| 221 | |
| 222 cc_values = [] | |
| 223 for cc in ccs: | |
| 224 cc_values.append({'name': cc}) | |
| 225 body['cc'] = cc_values | |
| 226 | |
| 227 body['labels'] = labels | |
| 228 | |
| 229 insertparams['body'] = body | |
| 230 | |
| 231 request = service.issues().insert(**insertparams) | |
| 232 result = request.execute() | |
| 233 | |
| 234 return result | |
| 235 | |
| 236 | |
| 237 # Skip buildbot runs of a CL if its commit log message contains the following | |
| 238 # substring. | |
| 239 SKIP_BUILDBOT_SUBSTRING = '(SkipBuildbotRuns)' | |
| 240 | |
| 241 # If the below regex is found in a CL's commit log message, only run the | |
| 242 # builders specified therein. | |
| 243 RUN_BUILDERS_REGEX = '\(RunBuilders:(.+)\)' | |
| 244 RUN_BUILDERS_RE_COMPILED = re.compile(RUN_BUILDERS_REGEX) | |
| 245 | |
| 246 | |
| 247 def CapWordsToUnderscores(string): | |
| 248 """ Converts a string containing capitalized words to one in which all | |
| 249 characters are lowercase and words are separated by underscores. | |
| 250 | |
| 251 Example: | |
| 252 'Nexus10' becomes 'nexus_10' | |
| 253 | |
| 254 string: string; string to manipulate. | |
| 255 """ | |
| 256 name_parts = [] | |
| 257 for part in re.split('(\d+)', string): | |
| 258 if re.match('(\d+)', part): | |
| 259 name_parts.append(part) | |
| 260 else: | |
| 261 name_parts.extend(re.findall('[A-Z][a-z]*', part)) | |
| 262 return '_'.join([part.lower() for part in name_parts]) | |
| 263 | |
| 264 | |
| 265 def UnderscoresToCapWords(string): | |
| 266 """ Converts a string lowercase words separated by underscores to one in which | |
| 267 words are capitalized and not separated by underscores. | |
| 268 | |
| 269 Example: | |
| 270 'nexus_10' becomes 'Nexus10' | |
| 271 | |
| 272 string: string; string to manipulate. | |
| 273 """ | |
| 274 name_parts = string.split('_') | |
| 275 return ''.join([part.title() for part in name_parts]) | |
| 276 | |
| 277 | |
| 278 # Since we can't modify the existing Helper class, we subclass it here, | |
| 279 # overriding the necessary parts to get things working as we want. | |
| 280 class SkiaHelper(object): | |
| 281 def __init__(self, defaults): | |
| 282 self._defaults = defaults | |
| 283 self._builders = [] | |
| 284 self._factories = {} | |
| 285 self._schedulers = {} | |
| 286 | |
| 287 def Builder(self, name, factory, gatekeeper=None, scheduler=None, | |
| 288 builddir=None, auto_reboot=False, notify_on_missing=False): | |
| 289 # Override the category with the first two parts of the builder name. | |
| 290 name_parts = name.split(builder_name_schema.BUILDER_NAME_SEP) | |
| 291 category = name_parts[0] | |
| 292 subcategory = name_parts[1] if len(name_parts) > 1 else 'default' | |
| 293 full_category = '|'.join((category, subcategory)) | |
| 294 self._builders.append({'name': name, | |
| 295 'factory': factory, | |
| 296 'gatekeeper': gatekeeper, | |
| 297 'schedulers': scheduler.split('|'), | |
| 298 'builddir': builddir, | |
| 299 'category': full_category, | |
| 300 'auto_reboot': auto_reboot, | |
| 301 'notify_on_missing': notify_on_missing}) | |
| 302 | |
| 303 def PeriodicScheduler(self, name, minute=0, hour='*', dayOfMonth='*', | |
| 304 month='*', dayOfWeek='*'): | |
| 305 """Helper method for the Periodic scheduler.""" | |
| 306 if name in self._schedulers: | |
| 307 raise ValueError('Scheduler %s already exists' % name) | |
| 308 self._schedulers[name] = {'type': 'PeriodicScheduler', | |
| 309 'builders': [], | |
| 310 'minute': minute, | |
| 311 'hour': hour, | |
| 312 'dayOfMonth': dayOfMonth, | |
| 313 'month': month, | |
| 314 'dayOfWeek': dayOfWeek} | |
| 315 | |
| 316 def Dependent(self, name, parent): | |
| 317 if name in self._schedulers: | |
| 318 raise ValueError('Scheduler %s already exists' % name) | |
| 319 self._schedulers[name] = {'type': 'Dependent', | |
| 320 'parent': parent, | |
| 321 'builders': []} | |
| 322 | |
| 323 def Factory(self, name, factory): | |
| 324 if name in self._factories: | |
| 325 raise ValueError('Factory %s already exists' % name) | |
| 326 self._factories[name] = factory | |
| 327 | |
| 328 def Scheduler(self, name, treeStableTimer=60, categories=None): | |
| 329 if name in self._schedulers: | |
| 330 raise ValueError('Scheduler %s already exists' % name) | |
| 331 self._schedulers[name] = {'type': 'Scheduler', | |
| 332 'treeStableTimer': treeStableTimer, | |
| 333 'builders': [], | |
| 334 'categories': categories} | |
| 335 | |
| 336 def TryJobSubversion(self, name): | |
| 337 """ Adds a Subversion-based try scheduler. """ | |
| 338 if name in self._schedulers: | |
| 339 raise ValueError('Scheduler %s already exists' % name) | |
| 340 self._schedulers[name] = {'type': 'TryJobSubversion', 'builders': []} | |
| 341 | |
| 342 def TryJobRietveld(self, name): | |
| 343 """ Adds a Rietveld-based try scheduler. """ | |
| 344 if name in self._schedulers: | |
| 345 raise ValueError('Scheduler %s already exists' % name) | |
| 346 self._schedulers[name] = {'type': 'TryJobRietveld', 'builders': []} | |
| 347 | |
| 348 def Update(self, c): | |
| 349 for builder in self._builders: | |
| 350 # Update the schedulers with the builder. | |
| 351 schedulers = builder['schedulers'] | |
| 352 if schedulers: | |
| 353 for scheduler in schedulers: | |
| 354 self._schedulers[scheduler]['builders'].append(builder['name']) | |
| 355 | |
| 356 # Construct the category. | |
| 357 categories = [] | |
| 358 if builder.get('category', None): | |
| 359 categories.append(builder['category']) | |
| 360 if builder.get('gatekeeper', None): | |
| 361 categories.extend(builder['gatekeeper'].split('|')) | |
| 362 category = '|'.join(categories) | |
| 363 | |
| 364 # Append the builder to the list. | |
| 365 new_builder = {'name': builder['name'], | |
| 366 'factory': self._factories[builder['factory']], | |
| 367 'category': category, | |
| 368 'auto_reboot': builder['auto_reboot']} | |
| 369 if builder['builddir']: | |
| 370 new_builder['builddir'] = builder['builddir'] | |
| 371 c['builders'].append(new_builder) | |
| 372 | |
| 373 c['builders'].sort(key=lambda builder: builder['name']) | |
| 374 | |
| 375 # Process the main schedulers. | |
| 376 for s_name in self._schedulers: | |
| 377 scheduler = self._schedulers[s_name] | |
| 378 if scheduler['type'] == 'Scheduler': | |
| 379 def filter_fn(change, builders): | |
| 380 """Filters out if change.comments contains certain keywords. | |
| 381 | |
| 382 The change is filtered out if the commit message contains: | |
| 383 * SKIP_BUILDBOT_SUBSTRING or | |
| 384 * RUN_BUILDERS_REGEX when the scheduler does not contain any of the | |
| 385 specified builders | |
| 386 | |
| 387 Args: | |
| 388 change: An instance of changes.Change. | |
| 389 builders: Sequence of strings. The builders that are run by this | |
| 390 scheduler. | |
| 391 | |
| 392 Returns: | |
| 393 If the change should be filtered out (i.e. not run by the buildbot | |
| 394 code) then False is returned else True is returned. | |
| 395 """ | |
| 396 if SKIP_BUILDBOT_SUBSTRING in change.comments: | |
| 397 return False | |
| 398 match_obj = RUN_BUILDERS_RE_COMPILED.search(change.comments) | |
| 399 if builders and match_obj: | |
| 400 for builder_to_run in match_obj.group(1).split(','): | |
| 401 if builder_to_run.strip() in builders: | |
| 402 break | |
| 403 else: | |
| 404 return False | |
| 405 return True | |
| 406 | |
| 407 skia_change_filter = SkiaChangeFilter( | |
| 408 builders=scheduler['builders'], | |
| 409 branch=skia_vars.GetGlobalVariable('master_branch_name'), | |
| 410 filter_fn=filter_fn) | |
| 411 | |
| 412 instance = Scheduler(name=s_name, | |
| 413 treeStableTimer=scheduler['treeStableTimer'], | |
| 414 builderNames=scheduler['builders'], | |
| 415 change_filter=skia_change_filter) | |
| 416 c['schedulers'].append(instance) | |
| 417 self._schedulers[s_name]['instance'] = instance | |
| 418 | |
| 419 # Process the periodic schedulers. | |
| 420 for s_name in self._schedulers: | |
| 421 scheduler = self._schedulers[s_name] | |
| 422 if scheduler['type'] == 'PeriodicScheduler': | |
| 423 instance = timed.Nightly( | |
| 424 name=s_name, | |
| 425 branch=skia_vars.GetGlobalVariable('master_branch_name'), | |
| 426 builderNames=scheduler['builders'], | |
| 427 minute=scheduler['minute'], | |
| 428 hour=scheduler['hour'], | |
| 429 dayOfMonth=scheduler['dayOfMonth'], | |
| 430 month=scheduler['month'], | |
| 431 dayOfWeek=scheduler['dayOfWeek']) | |
| 432 c['schedulers'].append(instance) | |
| 433 self._schedulers[s_name]['instance'] = instance | |
| 434 | |
| 435 # Process the Rietveld-based try schedulers. | |
| 436 for s_name in self._schedulers: | |
| 437 scheduler = self._schedulers[s_name] | |
| 438 if scheduler['type'] == 'TryJobRietveld': | |
| 439 pools = BuildersPools(s_name) | |
| 440 pools[s_name].extend(scheduler['builders']) | |
| 441 instance = try_job_rietveld.TryJobRietveld( | |
| 442 name=s_name, | |
| 443 pools=pools, | |
| 444 last_good_urls={'skia': None}, | |
| 445 code_review_sites={'skia': config_private.CODE_REVIEW_SITE}, | |
| 446 project='skia') | |
| 447 c['schedulers'].append(instance) | |
| 448 self._schedulers[s_name]['instance'] = instance | |
| 449 | |
| 450 # Process the svn-based try schedulers. | |
| 451 for s_name in self._schedulers: | |
| 452 scheduler = self._schedulers[s_name] | |
| 453 if scheduler['type'] == 'TryJobSubversion': | |
| 454 pools = BuildersPools(s_name) | |
| 455 pools[s_name].extend(scheduler['builders']) | |
| 456 instance = try_job_svn.TryJobSubversion( | |
| 457 name=s_name, | |
| 458 svn_url=TRY_SVN_BASEURL, | |
| 459 last_good_urls={'skia': None}, | |
| 460 code_review_sites={'skia': config_private.CODE_REVIEW_SITE}, | |
| 461 pools=pools) | |
| 462 c['schedulers'].append(instance) | |
| 463 self._schedulers[s_name]['instance'] = instance | |
| 464 | |
| 465 # Process the dependent schedulers. | |
| 466 for s_name in self._schedulers: | |
| 467 scheduler = self._schedulers[s_name] | |
| 468 if scheduler['type'] == 'Dependent': | |
| 469 instance = Dependent( | |
| 470 s_name, | |
| 471 self._schedulers[scheduler['parent']]['instance'], | |
| 472 scheduler['builders']) | |
| 473 c['schedulers'].append(instance) | |
| 474 self._schedulers[s_name]['instance'] = instance | |
| 475 | |
| 476 | |
| 477 def CanMergeBuildRequests(req1, req2): | |
| 478 """ Determine whether or not two BuildRequests can be merged. Note that the | |
| 479 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously | |
| 480 missing. This is because that method verifies that: | |
| 481 1. req1.source.repository == req2.source.repository | |
| 482 2. req1.source.project == req2.source.project | |
| 483 3. req1.source.branch == req2.source.branch | |
| 484 4. req1.patch == None and req2.patch = None | |
| 485 5. (req1.source.changes and req2.source.changes) or \ | |
| 486 (not req1.source.changes and not req2.source.changes and \ | |
| 487 req1.source.revision == req2.source.revision) | |
| 488 | |
| 489 Of the above, we want 1, 2, 3, and 5. | |
| 490 Instead of 4, we want to make sure that neither request is a Trybot request. | |
| 491 """ | |
| 492 # Verify that the repositories are the same (#1 above). | |
| 493 if req1.source.repository != req2.source.repository: | |
| 494 return False | |
| 495 | |
| 496 # Verify that the projects are the same (#2 above). | |
| 497 if req1.source.project != req2.source.project: | |
| 498 return False | |
| 499 | |
| 500 # Verify that the branches are the same (#3 above). | |
| 501 if req1.source.branch != req2.source.branch: | |
| 502 return False | |
| 503 | |
| 504 # If either is a try request, don't merge (#4 above). | |
| 505 if (builder_name_schema.IsTrybot(req1.buildername) or | |
| 506 builder_name_schema.IsTrybot(req2.buildername)): | |
| 507 return False | |
| 508 | |
| 509 # Verify that either: both requests are associated with changes OR neither | |
| 510 # request is associated with a change but the revisions match (#5 above). | |
| 511 if req1.source.changes and not req2.source.changes: | |
| 512 return False | |
| 513 if not req1.source.changes and req2.source.changes: | |
| 514 return False | |
| 515 if not (req1.source.changes and req2.source.changes): | |
| 516 if req1.source.revision != req2.source.revision: | |
| 517 return False | |
| 518 | |
| 519 return True | |
| 520 | |
| 521 | |
| 522 def get_current_revision(): | |
| 523 """Obtain the checked-out buildbot code revision.""" | |
| 524 checkout_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) | |
| 525 if os.path.isdir(os.path.join(checkout_dir, '.git')): | |
| 526 return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() | |
| 527 elif os.path.isdir(os.path.join(checkout_dir, '.svn')): | |
| 528 return subprocess.check_output(['svnversion', '.']).strip() | |
| 529 raise Exception('Unable to determine version control system.') | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 648353002:
Remove Skia's forked buildbot code (Closed)
Base URL: https://skia.googlesource.com/buildbot.git@master