| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file at | |
| 5 # http://src.chromium.org/viewvc/chrome/trunk/src/LICENSE | |
| 6 # This file is NOT under GPL. | |
| 7 | |
| 8 """Queries buildbot through the json interface. | |
| 9 """ | |
| 10 | |
| 11 __author__ = 'maruel@chromium.org' | |
| 12 __version__ = '1.2' | |
| 13 | |
| 14 import code | |
| 15 import datetime | |
| 16 import functools | |
| 17 import json | |
| 18 import logging | |
| 19 import optparse | |
| 20 import time | |
| 21 import urllib | |
| 22 import urllib2 | |
| 23 import sys | |
| 24 | |
| 25 try: | |
| 26 from natsort import natsorted | |
| 27 except ImportError: | |
| 28 # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted | |
| 29 # after "vm7". Defaults to normal sorting. | |
| 30 natsorted = sorted | |
| 31 | |
| 32 | |
| 33 # These values are buildbot constants used for Build and BuildStep. | |
| 34 # This line was copied from master/buildbot/status/builder.py. | |
| 35 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6) | |
| 36 | |
| 37 | |
| 38 ## Generic node caching code. | |
| 39 | |
| 40 | |
| 41 class Node(object): | |
| 42 """Root class for all nodes in the graph. | |
| 43 | |
| 44 Provides base functionality for any node in the graph, independent if it has | |
| 45 children or not or if its content can be addressed through an url or needs to | |
| 46 be fetched as part of another node. | |
| 47 | |
| 48 self.printable_attributes is only used for self documentation and for str() | |
| 49 implementation. | |
| 50 """ | |
| 51 printable_attributes = [] | |
| 52 | |
| 53 def __init__(self, parent, url): | |
| 54 self.printable_attributes = self.printable_attributes[:] | |
| 55 if url: | |
| 56 self.printable_attributes.append('url') | |
| 57 url = url.rstrip('/') | |
| 58 if parent is not None: | |
| 59 self.printable_attributes.append('parent') | |
| 60 self.url = url | |
| 61 self.parent = parent | |
| 62 | |
| 63 def __str__(self): | |
| 64 return self.to_string() | |
| 65 | |
| 66 def __repr__(self): | |
| 67 """Embeds key if present.""" | |
| 68 key = getattr(self, 'key', None) | |
| 69 if key is not None: | |
| 70 return '<%s key=%s>' % (self.__class__.__name__, key) | |
| 71 cached_keys = getattr(self, 'cached_keys', None) | |
| 72 if cached_keys is not None: | |
| 73 return '<%s keys=%s>' % (self.__class__.__name__, cached_keys) | |
| 74 return super(Node, self).__repr__() | |
| 75 | |
| 76 def to_string(self, maximum=100): | |
| 77 out = ['%s:' % self.__class__.__name__] | |
| 78 assert not 'printable_attributes' in self.printable_attributes | |
| 79 | |
| 80 def limit(txt): | |
| 81 txt = str(txt) | |
| 82 if maximum > 0: | |
| 83 if len(txt) > maximum + 2: | |
| 84 txt = txt[:maximum] + '...' | |
| 85 return txt | |
| 86 | |
| 87 for k in sorted(self.printable_attributes): | |
| 88 if k == 'parent': | |
| 89 # Avoid infinite recursion. | |
| 90 continue | |
| 91 out.append(limit(' %s: %r' % (k, getattr(self, k)))) | |
| 92 return '\n'.join(out) | |
| 93 | |
| 94 def refresh(self): | |
| 95 """Refreshes the data.""" | |
| 96 self.discard() | |
| 97 return self.cache() | |
| 98 | |
| 99 def cache(self): # pragma: no cover | |
| 100 """Caches the data.""" | |
| 101 raise NotImplementedError() | |
| 102 | |
| 103 def discard(self): # pragma: no cover | |
| 104 """Discards cached data. | |
| 105 | |
| 106 Pretty much everything is temporary except completed Build. | |
| 107 """ | |
| 108 raise NotImplementedError() | |
| 109 | |
| 110 def read_non_json(self, suburl): | |
| 111 """Returns raw data for a suburl. | |
| 112 | |
| 113 Contrary to self.read(), self.read_non_json() is always available since | |
| 114 suburl is rooted at the base url. read() is only accessible for resources | |
| 115 that have an URI. | |
| 116 """ | |
| 117 return self.parent.read_non_json(suburl) | |
| 118 | |
| 119 | |
| 120 class AddressableBaseDataNode(Node): # pylint: disable=W0223 | |
| 121 """A node that contains a dictionary of data that can be fetched with an url. | |
| 122 | |
| 123 The node is directly addressable. It also often can be fetched by the parent. | |
| 124 """ | |
| 125 printable_attributes = Node.printable_attributes + ['data'] | |
| 126 | |
| 127 def __init__(self, parent, url, data): | |
| 128 super(AddressableBaseDataNode, self).__init__(parent, url) | |
| 129 self._data = data | |
| 130 | |
| 131 @property | |
| 132 def cached_data(self): | |
| 133 return self._data | |
| 134 | |
| 135 @property | |
| 136 def data(self): | |
| 137 self.cache() | |
| 138 return self._data | |
| 139 | |
| 140 def cache(self): | |
| 141 if self._data is None: | |
| 142 self._data = self._readall() | |
| 143 return True | |
| 144 return False | |
| 145 | |
| 146 def discard(self): | |
| 147 self._data = None | |
| 148 | |
| 149 def read(self, suburl): | |
| 150 assert self.url, self.__class__.__name__ | |
| 151 url = self.url | |
| 152 if suburl: | |
| 153 url = '%s/%s' % (self.url, suburl) | |
| 154 return self.parent.read(url) | |
| 155 | |
| 156 def _readall(self): | |
| 157 return self.read('') | |
| 158 | |
| 159 | |
| 160 class AddressableDataNode(AddressableBaseDataNode): # pylint: disable=W0223 | |
| 161 """Automatically encodes the url.""" | |
| 162 | |
| 163 def __init__(self, parent, url, data): | |
| 164 super(AddressableDataNode, self).__init__(parent, urllib.quote(url), data) | |
| 165 | |
| 166 | |
| 167 class NonAddressableDataNode(Node): # pylint: disable=W0223 | |
| 168 """A node that cannot be addressed by an unique url. | |
| 169 | |
| 170 The data comes directly from the parent. | |
| 171 """ | |
| 172 def __init__(self, parent, subkey): | |
| 173 super(NonAddressableDataNode, self).__init__(parent, None) | |
| 174 self.subkey = subkey | |
| 175 | |
| 176 @property | |
| 177 def cached_data(self): | |
| 178 if self.parent.cached_data is None: | |
| 179 return None | |
| 180 return self.parent.cached_data[self.subkey] | |
| 181 | |
| 182 @property | |
| 183 def data(self): | |
| 184 return self.parent.data[self.subkey] | |
| 185 | |
| 186 def cache(self): | |
| 187 self.parent.cache() | |
| 188 | |
| 189 def discard(self): # pragma: no cover | |
| 190 """Avoid invalid state when parent recreate the object.""" | |
| 191 raise AttributeError('Call parent discard() instead') | |
| 192 | |
| 193 | |
| 194 class VirtualNodeList(Node): | |
| 195 """Base class for every node that has children. | |
| 196 | |
| 197 Adds partial supports for keys and iterator functionality. 'key' can be a | |
| 198 string or a int. Not to be used directly. | |
| 199 """ | |
| 200 printable_attributes = Node.printable_attributes + ['keys'] | |
| 201 | |
| 202 def __init__(self, parent, url): | |
| 203 super(VirtualNodeList, self).__init__(parent, url) | |
| 204 # Keeps the keys independently when ordering is needed. | |
| 205 self._is_cached = False | |
| 206 self._has_keys_cached = False | |
| 207 | |
| 208 def __contains__(self, key): | |
| 209 """Enables 'if i in obj:'.""" | |
| 210 return key in self.keys | |
| 211 | |
| 212 def __iter__(self): | |
| 213 """Enables 'for i in obj:'. It returns children.""" | |
| 214 self.cache_keys() | |
| 215 for key in self.keys: | |
| 216 yield self[key] | |
| 217 | |
| 218 def __len__(self): | |
| 219 """Enables 'len(obj)' to get the number of childs.""" | |
| 220 return len(self.keys) | |
| 221 | |
| 222 def discard(self): | |
| 223 """Discards data. | |
| 224 | |
| 225 The default behavior is to not invalidate cached keys. The only place where | |
| 226 keys need to be invalidated is with Builds. | |
| 227 """ | |
| 228 self._is_cached = False | |
| 229 self._has_keys_cached = False | |
| 230 | |
| 231 @property | |
| 232 def cached_children(self): # pragma: no cover | |
| 233 """Returns an iterator over the children that are cached.""" | |
| 234 raise NotImplementedError() | |
| 235 | |
| 236 @property | |
| 237 def cached_keys(self): # pragma: no cover | |
| 238 raise NotImplementedError() | |
| 239 | |
| 240 @property | |
| 241 def keys(self): # pragma: no cover | |
| 242 """Returns the keys for every children.""" | |
| 243 raise NotImplementedError() | |
| 244 | |
| 245 def __getitem__(self, key): # pragma: no cover | |
| 246 """Returns a child, without fetching its data. | |
| 247 | |
| 248 The children could be invalid since no verification is done. | |
| 249 """ | |
| 250 raise NotImplementedError() | |
| 251 | |
| 252 def cache(self): # pragma: no cover | |
| 253 """Cache all the children.""" | |
| 254 raise NotImplementedError() | |
| 255 | |
| 256 def cache_keys(self): # pragma: no cover | |
| 257 """Cache all children's keys.""" | |
| 258 raise NotImplementedError() | |
| 259 | |
| 260 | |
| 261 class NodeList(VirtualNodeList): # pylint: disable=W0223 | |
| 262 """Adds a cache of the keys.""" | |
| 263 def __init__(self, parent, url): | |
| 264 super(NodeList, self).__init__(parent, url) | |
| 265 self._keys = [] | |
| 266 | |
| 267 @property | |
| 268 def cached_keys(self): | |
| 269 return self._keys | |
| 270 | |
| 271 @property | |
| 272 def keys(self): | |
| 273 self.cache_keys() | |
| 274 return self._keys | |
| 275 | |
| 276 | |
| 277 class NonAddressableNodeList(VirtualNodeList): # pylint: disable=W0223 | |
| 278 """A node that contains children but retrieves all its data from its parent. | |
| 279 | |
| 280 I.e. there's no url to get directly this data. | |
| 281 """ | |
| 282 # Child class object for children of this instance. For example, BuildSteps | |
| 283 # has BuildStep children. | |
| 284 _child_cls = None | |
| 285 | |
| 286 def __init__(self, parent, subkey): | |
| 287 super(NonAddressableNodeList, self).__init__(parent, None) | |
| 288 self.subkey = subkey | |
| 289 assert ( | |
| 290 not isinstance(self._child_cls, NonAddressableDataNode) and | |
| 291 issubclass(self._child_cls, NonAddressableDataNode)), ( | |
| 292 self._child_cls.__name__) | |
| 293 | |
| 294 @property | |
| 295 def cached_children(self): | |
| 296 if self.parent.cached_data is not None: | |
| 297 for i in xrange(len(self.parent.cached_data[self.subkey])): | |
| 298 yield self[i] | |
| 299 | |
| 300 @property | |
| 301 def cached_data(self): | |
| 302 if self.parent.cached_data is None: | |
| 303 return None | |
| 304 return self.parent.data.get(self.subkey, None) | |
| 305 | |
| 306 @property | |
| 307 def cached_keys(self): | |
| 308 if self.parent.cached_data is None: | |
| 309 return None | |
| 310 return range(len(self.parent.data.get(self.subkey, []))) | |
| 311 | |
| 312 @property | |
| 313 def data(self): | |
| 314 return self.parent.data[self.subkey] | |
| 315 | |
| 316 def cache(self): | |
| 317 self.parent.cache() | |
| 318 | |
| 319 def cache_keys(self): | |
| 320 self.parent.cache() | |
| 321 | |
| 322 def discard(self): # pragma: no cover | |
| 323 """Avoid infinite recursion by having the caller calls the parent's | |
| 324 discard() explicitely. | |
| 325 """ | |
| 326 raise AttributeError('Call parent discard() instead') | |
| 327 | |
| 328 def __iter__(self): | |
| 329 """Enables 'for i in obj:'. It returns children.""" | |
| 330 if self.data: | |
| 331 for i in xrange(len(self.data)): | |
| 332 yield self[i] | |
| 333 | |
| 334 def __getitem__(self, key): | |
| 335 """Doesn't cache the value, it's not needed. | |
| 336 | |
| 337 TODO(maruel): Cache? | |
| 338 """ | |
| 339 if isinstance(key, int) and key < 0: | |
| 340 key = len(self.data) + key | |
| 341 # pylint: disable=E1102 | |
| 342 return self._child_cls(self, key) | |
| 343 | |
| 344 | |
| 345 class AddressableNodeList(NodeList): | |
| 346 """A node that has children that can be addressed with an url.""" | |
| 347 | |
| 348 # Child class object for children of this instance. For example, Builders has | |
| 349 # Builder children and Builds has Build children. | |
| 350 _child_cls = None | |
| 351 | |
| 352 def __init__(self, parent, url): | |
| 353 super(AddressableNodeList, self).__init__(parent, url) | |
| 354 self._cache = {} | |
| 355 assert ( | |
| 356 not isinstance(self._child_cls, AddressableDataNode) and | |
| 357 issubclass(self._child_cls, AddressableDataNode)), ( | |
| 358 self._child_cls.__name__) | |
| 359 | |
| 360 @property | |
| 361 def cached_children(self): | |
| 362 for item in self._cache.itervalues(): | |
| 363 if item.cached_data is not None: | |
| 364 yield item | |
| 365 | |
| 366 @property | |
| 367 def cached_keys(self): | |
| 368 return self._cache.keys() | |
| 369 | |
| 370 def __getitem__(self, key): | |
| 371 """Enables 'obj[i]'.""" | |
| 372 if self._has_keys_cached and not key in self._keys: | |
| 373 raise KeyError(key) | |
| 374 | |
| 375 if not key in self._cache: | |
| 376 # Create an empty object. | |
| 377 self._create_obj(key, None) | |
| 378 return self._cache[key] | |
| 379 | |
| 380 def cache(self): | |
| 381 if not self._is_cached: | |
| 382 data = self._readall() | |
| 383 for key in sorted(data): | |
| 384 self._create_obj(key, data[key]) | |
| 385 self._is_cached = True | |
| 386 self._has_keys_cached = True | |
| 387 | |
| 388 def cache_partial(self, children): | |
| 389 """Caches a partial number of children. | |
| 390 | |
| 391 This method is more efficient since it does a single request for all the | |
| 392 children instead of one request per children. | |
| 393 | |
| 394 It only grab objects not already cached. | |
| 395 """ | |
| 396 # pylint: disable=W0212 | |
| 397 if not self._is_cached: | |
| 398 to_fetch = [ | |
| 399 child for child in children | |
| 400 if not (child in self._cache and self._cache[child].cached_data) | |
| 401 ] | |
| 402 if to_fetch: | |
| 403 # Similar to cache(). The only reason to sort is to simplify testing. | |
| 404 params = '&'.join( | |
| 405 'select=%s' % urllib.quote(str(v)) for v in sorted(to_fetch)) | |
| 406 data = self.read('?' + params) | |
| 407 for key in sorted(data): | |
| 408 self._create_obj(key, data[key]) | |
| 409 | |
| 410 def cache_keys(self): | |
| 411 """Implement to speed up enumeration. Defaults to call cache().""" | |
| 412 if not self._has_keys_cached: | |
| 413 self.cache() | |
| 414 assert self._has_keys_cached | |
| 415 | |
| 416 def discard(self): | |
| 417 """Discards temporary children.""" | |
| 418 super(AddressableNodeList, self).discard() | |
| 419 for v in self._cache.itervalues(): | |
| 420 v.discard() | |
| 421 | |
| 422 def read(self, suburl): | |
| 423 assert self.url, self.__class__.__name__ | |
| 424 url = self.url | |
| 425 if suburl: | |
| 426 url = '%s/%s' % (self.url, suburl) | |
| 427 return self.parent.read(url) | |
| 428 | |
| 429 def _create_obj(self, key, data): | |
| 430 """Creates an object of type self._child_cls.""" | |
| 431 # pylint: disable=E1102 | |
| 432 obj = self._child_cls(self, key, data) | |
| 433 # obj.key and key may be different. | |
| 434 # No need to overide cached data with None. | |
| 435 if data is not None or obj.key not in self._cache: | |
| 436 self._cache[obj.key] = obj | |
| 437 if obj.key not in self._keys: | |
| 438 self._keys.append(obj.key) | |
| 439 | |
| 440 def _readall(self): | |
| 441 return self.read('') | |
| 442 | |
| 443 | |
| 444 class SubViewNodeList(VirtualNodeList): # pylint: disable=W0223 | |
| 445 """A node that shows a subset of children that comes from another structure. | |
| 446 | |
| 447 The node is not addressable. | |
| 448 | |
| 449 E.g. the keys are retrieved from parent but the actual data comes from | |
| 450 virtual_parent. | |
| 451 """ | |
| 452 | |
| 453 def __init__(self, parent, virtual_parent, subkey): | |
| 454 super(SubViewNodeList, self).__init__(parent, None) | |
| 455 self.subkey = subkey | |
| 456 self.virtual_parent = virtual_parent | |
| 457 assert isinstance(self.parent, AddressableDataNode) | |
| 458 assert isinstance(self.virtual_parent, NodeList) | |
| 459 | |
| 460 @property | |
| 461 def cached_children(self): | |
| 462 if self.parent.cached_data is not None: | |
| 463 for item in self.keys: | |
| 464 if item in self.virtual_parent.keys: | |
| 465 child = self[item] | |
| 466 if child.cached_data is not None: | |
| 467 yield child | |
| 468 | |
| 469 @property | |
| 470 def cached_keys(self): | |
| 471 return (self.parent.cached_data or {}).get(self.subkey, []) | |
| 472 | |
| 473 @property | |
| 474 def keys(self): | |
| 475 self.cache_keys() | |
| 476 return self.parent.data.get(self.subkey, []) | |
| 477 | |
| 478 def cache(self): | |
| 479 """Batch request for each child in a single read request.""" | |
| 480 if not self._is_cached: | |
| 481 self.virtual_parent.cache_partial(self.keys) | |
| 482 self._is_cached = True | |
| 483 | |
| 484 def cache_keys(self): | |
| 485 if not self._has_keys_cached: | |
| 486 self.parent.cache() | |
| 487 self._has_keys_cached = True | |
| 488 | |
| 489 def discard(self): | |
| 490 if self.parent.cached_data is not None: | |
| 491 for child in self.virtual_parent.cached_children: | |
| 492 if child.key in self.keys: | |
| 493 child.discard() | |
| 494 self.parent.discard() | |
| 495 super(SubViewNodeList, self).discard() | |
| 496 | |
| 497 def __getitem__(self, key): | |
| 498 """Makes sure the key is in our key but grab it from the virtual parent.""" | |
| 499 return self.virtual_parent[key] | |
| 500 | |
| 501 def __iter__(self): | |
| 502 self.cache() | |
| 503 return super(SubViewNodeList, self).__iter__() | |
| 504 | |
| 505 | |
| 506 ############################################################################### | |
| 507 ## Buildbot-specific code | |
| 508 | |
| 509 | |
| 510 class Slave(AddressableDataNode): | |
| 511 printable_attributes = AddressableDataNode.printable_attributes + [ | |
| 512 'name', 'key', 'connected', 'version', | |
| 513 ] | |
| 514 | |
| 515 def __init__(self, parent, name, data): | |
| 516 super(Slave, self).__init__(parent, name, data) | |
| 517 self.name = name | |
| 518 self.key = self.name | |
| 519 # TODO(maruel): Add SlaveBuilders and a 'builders' property. | |
| 520 # TODO(maruel): Add a 'running_builds' property. | |
| 521 | |
| 522 @property | |
| 523 def connected(self): | |
| 524 return self.data.get('connected', False) | |
| 525 | |
| 526 @property | |
| 527 def version(self): | |
| 528 return self.data.get('version') | |
| 529 | |
| 530 | |
| 531 class Slaves(AddressableNodeList): | |
| 532 _child_cls = Slave | |
| 533 printable_attributes = AddressableNodeList.printable_attributes + ['names'] | |
| 534 | |
| 535 def __init__(self, parent): | |
| 536 super(Slaves, self).__init__(parent, 'slaves') | |
| 537 | |
| 538 @property | |
| 539 def names(self): | |
| 540 return self.keys | |
| 541 | |
| 542 | |
| 543 class BuilderSlaves(SubViewNodeList): | |
| 544 """Similar to Slaves but only list slaves connected to a specific builder. | |
| 545 """ | |
| 546 printable_attributes = SubViewNodeList.printable_attributes + ['names'] | |
| 547 | |
| 548 def __init__(self, parent): | |
| 549 super(BuilderSlaves, self).__init__( | |
| 550 parent, parent.parent.parent.slaves, 'slaves') | |
| 551 | |
| 552 @property | |
| 553 def names(self): | |
| 554 return self.keys | |
| 555 | |
| 556 | |
| 557 class BuildStep(NonAddressableDataNode): | |
| 558 printable_attributes = NonAddressableDataNode.printable_attributes + [ | |
| 559 'name', 'number', 'start_time', 'end_time', 'duration', 'is_started', | |
| 560 'is_finished', 'is_running', | |
| 561 'result', 'simplified_result', | |
| 562 ] | |
| 563 | |
| 564 def __init__(self, parent, number): | |
| 565 """It's already pre-loaded by definition since the data is retrieve via the | |
| 566 Build object. | |
| 567 """ | |
| 568 assert isinstance(number, int) | |
| 569 super(BuildStep, self).__init__(parent, number) | |
| 570 self.number = number | |
| 571 | |
| 572 @property | |
| 573 def build(self): | |
| 574 """Returns the Build object for this BuildStep.""" | |
| 575 # Build.BuildSteps.BuildStep | |
| 576 return self.parent.parent | |
| 577 | |
| 578 @property | |
| 579 def start_time(self): | |
| 580 if self.data.get('times'): | |
| 581 return int(round(self.data['times'][0])) | |
| 582 | |
| 583 @property | |
| 584 def end_time(self): | |
| 585 times = self.data.get('times') | |
| 586 if times and len(times) == 2 and times[1]: | |
| 587 return int(round(times[1])) | |
| 588 | |
| 589 @property | |
| 590 def duration(self): | |
| 591 if self.start_time: | |
| 592 return (self.end_time or int(round(time.time()))) - self.start_time | |
| 593 | |
| 594 @property | |
| 595 def name(self): | |
| 596 return self.data['name'] | |
| 597 | |
| 598 @property | |
| 599 def is_started(self): | |
| 600 return self.data.get('isStarted', False) | |
| 601 | |
| 602 @property | |
| 603 def is_finished(self): | |
| 604 return self.data.get('isFinished', False) | |
| 605 | |
| 606 @property | |
| 607 def is_running(self): | |
| 608 return self.is_started and not self.is_finished | |
| 609 | |
| 610 @property | |
| 611 def result(self): | |
| 612 result = self.data.get('results') | |
| 613 if result is None: | |
| 614 # results may be 0, in that case with filter=1, the value won't be | |
| 615 # present. | |
| 616 if self.data.get('isFinished'): | |
| 617 result = self.data.get('results', 0) | |
| 618 while isinstance(result, list): | |
| 619 result = result[0] | |
| 620 return result | |
| 621 | |
| 622 @property | |
| 623 def simplified_result(self): | |
| 624 """Returns a simplified 3 state value, True, False or None.""" | |
| 625 result = self.result | |
| 626 if result in (SUCCESS, WARNINGS): | |
| 627 return True | |
| 628 elif result in (FAILURE, EXCEPTION, RETRY): | |
| 629 return False | |
| 630 assert result in (None, SKIPPED), (result, self.data) | |
| 631 return None | |
| 632 | |
| 633 @property | |
| 634 def stdio(self): | |
| 635 """Returns the stdio for this step or None if not available.""" | |
| 636 # Parents ordering is BuildSteps / Build / Builds / Builders | |
| 637 # A bit hackish but works. | |
| 638 build = self.build | |
| 639 builder = build.builder | |
| 640 suburl = 'builders/%s/builds/%d/steps/%s/logs/stdio/text' % ( | |
| 641 builder.name, build.number, self.name) | |
| 642 return self.read_non_json(suburl) | |
| 643 | |
| 644 | |
| 645 class BuildSteps(NonAddressableNodeList): | |
| 646 """Duplicates keys to support lookup by both step number and step name.""" | |
| 647 printable_attributes = NonAddressableNodeList.printable_attributes + [ | |
| 648 'failed', | |
| 649 ] | |
| 650 _child_cls = BuildStep | |
| 651 | |
| 652 def __init__(self, parent): | |
| 653 """It's already pre-loaded by definition since the data is retrieve via the | |
| 654 Build object. | |
| 655 """ | |
| 656 super(BuildSteps, self).__init__(parent, 'steps') | |
| 657 | |
| 658 @property | |
| 659 def keys(self): | |
| 660 """Returns the steps name in order.""" | |
| 661 return [i['name'] for i in (self.data or [])] | |
| 662 | |
| 663 @property | |
| 664 def failed(self): | |
| 665 """Shortcuts that lists the step names of steps that failed.""" | |
| 666 return [step.name for step in self if step.simplified_result is False] | |
| 667 | |
| 668 def __getitem__(self, key): | |
| 669 """Accept step name in addition to index number.""" | |
| 670 if isinstance(key, basestring): | |
| 671 # It's a string, try to find the corresponding index. | |
| 672 for i, step in enumerate(self.data): | |
| 673 if step['name'] == key: | |
| 674 key = i | |
| 675 break | |
| 676 else: | |
| 677 raise KeyError(key) | |
| 678 return super(BuildSteps, self).__getitem__(key) | |
| 679 | |
| 680 | |
| 681 class Build(AddressableDataNode): | |
| 682 printable_attributes = AddressableDataNode.printable_attributes + [ | |
| 683 'key', 'number', 'steps', 'blame', 'reason', 'revision', 'result', | |
| 684 'simplified_result', 'start_time', 'end_time', 'duration', 'slave', | |
| 685 'properties', 'completed', | |
| 686 ] | |
| 687 | |
| 688 def __init__(self, parent, key, data): | |
| 689 super(Build, self).__init__(parent, str(key), data) | |
| 690 self.number = int(key) | |
| 691 self.key = self.number | |
| 692 self.steps = BuildSteps(self) | |
| 693 | |
| 694 @property | |
| 695 def blame(self): | |
| 696 return self.data.get('blame', []) | |
| 697 | |
| 698 @property | |
| 699 def builder(self): | |
| 700 """Returns the Builder object. | |
| 701 | |
| 702 Goes up the hierarchy to find the Buildbot.builders[builder] instance. | |
| 703 """ | |
| 704 return self.parent.parent.parent.parent.builders[self.data['builderName']] | |
| 705 | |
| 706 @property | |
| 707 def start_time(self): | |
| 708 if self.data.get('times'): | |
| 709 return int(round(self.data['times'][0])) | |
| 710 | |
| 711 @property | |
| 712 def end_time(self): | |
| 713 times = self.data.get('times') | |
| 714 if times and len(times) == 2 and times[1]: | |
| 715 return int(round(times[1])) | |
| 716 | |
| 717 @property | |
| 718 def duration(self): | |
| 719 if self.start_time: | |
| 720 return (self.end_time or int(round(time.time()))) - self.start_time | |
| 721 | |
| 722 @property | |
| 723 def eta(self): | |
| 724 return self.data.get('eta', 0) | |
| 725 | |
| 726 @property | |
| 727 def completed(self): | |
| 728 return self.data.get('currentStep') is None | |
| 729 | |
| 730 @property | |
| 731 def properties(self): | |
| 732 return self.data.get('properties', []) | |
| 733 | |
| 734 @property | |
| 735 def properties_as_dict(self): | |
| 736 """Converts the 3-tuple properties into a dict(p[0]: p[1]) and ignores the | |
| 737 property's source. | |
| 738 """ | |
| 739 return dict((p[0], p[1]) for p in self.properties) | |
| 740 | |
| 741 @property | |
| 742 def reason(self): | |
| 743 return self.data.get('reason') | |
| 744 | |
| 745 @property | |
| 746 def result(self): | |
| 747 result = self.data.get('results') | |
| 748 while isinstance(result, list): | |
| 749 result = result[0] | |
| 750 if result is None and self.steps: | |
| 751 # results may be 0, in that case with filter=1, the value won't be | |
| 752 # present. | |
| 753 result = self.steps[-1].result | |
| 754 return result | |
| 755 | |
| 756 @property | |
| 757 def revision(self): | |
| 758 return self.data.get('sourceStamp', {}).get('revision') | |
| 759 | |
| 760 @property | |
| 761 def simplified_result(self): | |
| 762 """Returns a simplified 3 state value, True, False or None.""" | |
| 763 result = self.result | |
| 764 if result in (SUCCESS, WARNINGS, SKIPPED): | |
| 765 return True | |
| 766 elif result in (FAILURE, EXCEPTION, RETRY): | |
| 767 return False | |
| 768 assert result is None, (result, self.data) | |
| 769 return None | |
| 770 | |
| 771 @property | |
| 772 def slave(self): | |
| 773 """Returns the Slave object. | |
| 774 | |
| 775 Goes up the hierarchy to find the Buildbot.slaves[slave] instance. | |
| 776 """ | |
| 777 return self.parent.parent.parent.parent.slaves[self.data['slave']] | |
| 778 | |
| 779 def discard(self): | |
| 780 """Completed Build isn't discarded.""" | |
| 781 if self._data and self.result is None: | |
| 782 assert not self.steps or not self.steps[-1].data.get('isFinished') | |
| 783 self._data = None | |
| 784 | |
| 785 | |
| 786 class CurrentBuilds(SubViewNodeList): | |
| 787 """Lists of the current builds.""" | |
| 788 def __init__(self, parent): | |
| 789 super(CurrentBuilds, self).__init__( | |
| 790 parent, parent.builds, 'currentBuilds') | |
| 791 | |
| 792 | |
| 793 class PendingBuilds(AddressableDataNode): | |
| 794 def __init__(self, parent): | |
| 795 super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None) | |
| 796 | |
| 797 | |
| 798 class Builds(AddressableNodeList): | |
| 799 """Supports iteration. | |
| 800 | |
| 801 Recommends using .cache() to speed up if a significant number of builds are | |
| 802 iterated over. | |
| 803 """ | |
| 804 _child_cls = Build | |
| 805 | |
| 806 def __init__(self, parent): | |
| 807 super(Builds, self).__init__(parent, 'builds') | |
| 808 | |
| 809 def __getitem__(self, key): | |
| 810 """Adds supports for negative reference and enables retrieving non-cached | |
| 811 builds. | |
| 812 | |
| 813 e.g. -1 is the last build, -2 is the previous build before the last one. | |
| 814 """ | |
| 815 key = int(key) | |
| 816 if key < 0: | |
| 817 # Convert negative to positive build number. | |
| 818 self.cache_keys() | |
| 819 # Since the negative value can be outside of the cache keys range, use the | |
| 820 # highest key value and calculate from it. | |
| 821 key = max(self._keys) + key + 1 | |
| 822 | |
| 823 if not key in self._cache: | |
| 824 # Create an empty object. | |
| 825 self._create_obj(key, None) | |
| 826 return self._cache[key] | |
| 827 | |
| 828 def __iter__(self): | |
| 829 """Returns cached Build objects in reversed order. | |
| 830 | |
| 831 The most recent build is returned first and then in reverse chronological | |
| 832 order, up to the oldest cached build by the server. Older builds can be | |
| 833 accessed but will trigger significantly more I/O so they are not included by | |
| 834 default in the iteration. | |
| 835 | |
| 836 To access the older builds, use self.iterall() instead. | |
| 837 """ | |
| 838 self.cache() | |
| 839 return reversed(self._cache.values()) | |
| 840 | |
| 841 def iterall(self): | |
| 842 """Returns Build objects in decreasing order unbounded up to build 0. | |
| 843 | |
| 844 The most recent build is returned first and then in reverse chronological | |
| 845 order. Older builds can be accessed and will trigger significantly more I/O | |
| 846 so use this carefully. | |
| 847 """ | |
| 848 # Only cache keys here. | |
| 849 self.cache_keys() | |
| 850 if self._keys: | |
| 851 for i in xrange(max(self._keys), -1, -1): | |
| 852 yield self[i] | |
| 853 | |
| 854 def cache_keys(self): | |
| 855 """Grabs the keys (build numbers) from the builder.""" | |
| 856 if not self._has_keys_cached: | |
| 857 for i in self.parent.data.get('cachedBuilds', []): | |
| 858 i = int(i) | |
| 859 self._cache.setdefault(i, Build(self, i, None)) | |
| 860 if i not in self._keys: | |
| 861 self._keys.append(i) | |
| 862 self._has_keys_cached = True | |
| 863 | |
| 864 def discard(self): | |
| 865 super(Builds, self).discard() | |
| 866 # Can't keep keys. | |
| 867 self._has_keys_cached = False | |
| 868 | |
| 869 def _readall(self): | |
| 870 return self.read('_all') | |
| 871 | |
| 872 | |
| 873 class Builder(AddressableDataNode): | |
| 874 printable_attributes = AddressableDataNode.printable_attributes + [ | |
| 875 'name', 'key', 'builds', 'slaves', 'pending_builds', 'current_builds', | |
| 876 ] | |
| 877 | |
| 878 def __init__(self, parent, name, data): | |
| 879 super(Builder, self).__init__(parent, name, data) | |
| 880 self.name = name | |
| 881 self.key = name | |
| 882 self.builds = Builds(self) | |
| 883 self.slaves = BuilderSlaves(self) | |
| 884 self.current_builds = CurrentBuilds(self) | |
| 885 self.pending_builds = PendingBuilds(self) | |
| 886 | |
| 887 def discard(self): | |
| 888 super(Builder, self).discard() | |
| 889 self.builds.discard() | |
| 890 self.slaves.discard() | |
| 891 self.current_builds.discard() | |
| 892 | |
| 893 | |
| 894 class Builders(AddressableNodeList): | |
| 895 """Root list of builders.""" | |
| 896 _child_cls = Builder | |
| 897 | |
| 898 def __init__(self, parent): | |
| 899 super(Builders, self).__init__(parent, 'builders') | |
| 900 | |
| 901 | |
| 902 class Buildbot(AddressableBaseDataNode): | |
| 903 """If a master restart occurs, this object should be recreated as it caches | |
| 904 data. | |
| 905 """ | |
| 906 # Throttle fetches to not kill the server. | |
| 907 auto_throttle = None | |
| 908 printable_attributes = AddressableDataNode.printable_attributes + [ | |
| 909 'slaves', 'builders', 'last_fetch', | |
| 910 ] | |
| 911 | |
| 912 def __init__(self, url): | |
| 913 super(Buildbot, self).__init__(None, url, None) | |
| 914 self._builders = Builders(self) | |
| 915 self._slaves = Slaves(self) | |
| 916 self.last_fetch = None | |
| 917 | |
| 918 @property | |
| 919 def builders(self): | |
| 920 return self._builders | |
| 921 | |
| 922 @property | |
| 923 def slaves(self): | |
| 924 return self._slaves | |
| 925 | |
| 926 def discard(self): | |
| 927 """Discards information about Builders and Slaves.""" | |
| 928 super(Buildbot, self).discard() | |
| 929 self._builders.discard() | |
| 930 self._slaves.discard() | |
| 931 | |
| 932 def read(self, suburl): | |
| 933 """Returns json decoded data for the suburl.""" | |
| 934 if self.auto_throttle: | |
| 935 if self.last_fetch: | |
| 936 delta = datetime.datetime.utcnow() - self.last_fetch | |
| 937 remaining = (datetime.timedelta(seconds=self.auto_throttle) - | |
| 938 delta) | |
| 939 if remaining > datetime.timedelta(seconds=0): | |
| 940 logging.debug('Sleeping for %ss' % remaining) | |
| 941 time.sleep(remaining.seconds) | |
| 942 self.last_fetch = datetime.datetime.utcnow() | |
| 943 url = '%s/json/%s' % (self.url, suburl) | |
| 944 if '?' in url: | |
| 945 url += '&filter=1' | |
| 946 else: | |
| 947 url += '?filter=1' | |
| 948 logging.debug('read(%s)' % suburl) | |
| 949 try: | |
| 950 channel = urllib.urlopen(url) | |
| 951 data = channel.read() | |
| 952 except IOError as e: | |
| 953 logging.warning('caught %s while fetching "%s"; re-throwing' % ( | |
| 954 str(e), url)) | |
| 955 raise | |
| 956 try: | |
| 957 return json.loads(data) | |
| 958 except ValueError: | |
| 959 if channel.getcode() >= 400: | |
| 960 # Convert it into an HTTPError for easier processing. | |
| 961 raise urllib2.HTTPError( | |
| 962 url, channel.getcode(), '%s:\n%s' % (url, data), channel.headers, | |
| 963 None) | |
| 964 raise | |
| 965 | |
| 966 def read_non_json(self, suburl): | |
| 967 """Returns data for an arbitrary suburl outside of the /json/ path.""" | |
| 968 logging.debug('read_non_json(%s)' % suburl) | |
| 969 return urllib.urlopen('%s/%s' % (self.url, suburl)).read() | |
| 970 | |
| 971 def _readall(self): | |
| 972 return self.read('project') | |
| 973 | |
| 974 | |
| 975 ############################################################################### | |
| 976 ## Controller code | |
| 977 | |
| 978 | |
| 979 def usage(more): | |
| 980 def hook(fn): | |
| 981 fn.func_usage_more = more | |
| 982 return fn | |
| 983 return hook | |
| 984 | |
| 985 | |
| 986 def need_buildbot(fn): | |
| 987 """Post-parse args to create a buildbot object.""" | |
| 988 @functools.wraps(fn) | |
| 989 def hook(parser, args, *extra_args, **kwargs): | |
| 990 old_parse_args = parser.parse_args | |
| 991 def new_parse_args(args): | |
| 992 options, args = old_parse_args(args) | |
| 993 if len(args) < 1: | |
| 994 parser.error('Need to pass the root url of the buildbot') | |
| 995 url = args.pop(0) | |
| 996 if not url.startswith('http'): | |
| 997 url = 'http://' + url | |
| 998 buildbot = Buildbot(url) | |
| 999 buildbot.auto_throttle = options.throttle | |
| 1000 return options, args, buildbot | |
| 1001 parser.parse_args = new_parse_args | |
| 1002 # Call the original function with the modified parser. | |
| 1003 return fn(parser, args, *extra_args, **kwargs) | |
| 1004 | |
| 1005 hook.func_usage_more = '[options] <url>' | |
| 1006 return hook | |
| 1007 | |
| 1008 | |
| 1009 @need_buildbot | |
| 1010 def CMDpending(parser, args): | |
| 1011 """Lists pending jobs.""" | |
| 1012 parser.add_option( | |
| 1013 '-b', '--builder', dest='builders', action='append', default=[], | |
| 1014 help='Builders to filter on') | |
| 1015 options, args, buildbot = parser.parse_args(args) | |
| 1016 if args: | |
| 1017 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1018 if not options.builders: | |
| 1019 options.builders = buildbot.builders.keys | |
| 1020 for builder in options.builders: | |
| 1021 builder = buildbot.builders[builder] | |
| 1022 pending_builds = builder.data.get('pendingBuilds', 0) | |
| 1023 if not pending_builds: | |
| 1024 continue | |
| 1025 print 'Builder %s: %d' % (builder.name, pending_builds) | |
| 1026 if not options.quiet: | |
| 1027 for pending in builder.pending_builds.data: | |
| 1028 if 'revision' in pending['source']: | |
| 1029 print ' revision: %s' % pending['source']['revision'] | |
| 1030 for change in pending['source']['changes']: | |
| 1031 print ' change:' | |
| 1032 print ' comment: %r' % unicode(change['comments'][:50]) | |
| 1033 print ' who: %s' % change['who'] | |
| 1034 return 0 | |
| 1035 | |
| 1036 | |
| 1037 @usage('[options] <url> [commands] ...') | |
| 1038 @need_buildbot | |
| 1039 def CMDrun(parser, args): | |
| 1040 """Runs commands passed as parameters. | |
| 1041 | |
| 1042 When passing commands on the command line, each command will be run as if it | |
| 1043 was on its own line. | |
| 1044 """ | |
| 1045 parser.add_option('-f', '--file', help='Read script from file') | |
| 1046 parser.add_option( | |
| 1047 '-i', dest='use_stdin', action='store_true', help='Read script on stdin') | |
| 1048 # Variable 'buildbot' is not used directly. | |
| 1049 # pylint: disable=W0612 | |
| 1050 options, args, buildbot = parser.parse_args(args) | |
| 1051 if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1: | |
| 1052 parser.error('Need to pass only one of: <commands>, -f <file> or -i') | |
| 1053 if options.use_stdin: | |
| 1054 cmds = sys.stdin.read() | |
| 1055 elif options.file: | |
| 1056 cmds = open(options.file).read() | |
| 1057 else: | |
| 1058 cmds = '\n'.join(args) | |
| 1059 compiled = compile(cmds, '<cmd line>', 'exec') | |
| 1060 eval(compiled, globals(), locals()) | |
| 1061 return 0 | |
| 1062 | |
| 1063 | |
| 1064 @need_buildbot | |
| 1065 def CMDinteractive(parser, args): | |
| 1066 """Runs an interactive shell to run queries.""" | |
| 1067 _, args, buildbot = parser.parse_args(args) | |
| 1068 if args: | |
| 1069 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1070 prompt = ( | |
| 1071 'Buildbot interactive console for "%s".\n' | |
| 1072 'Hint: Start with typing: \'buildbot.printable_attributes\' or ' | |
| 1073 '\'print str(buildbot)\' to explore.') % buildbot.url | |
| 1074 local_vars = { | |
| 1075 'buildbot': buildbot, | |
| 1076 'b': buildbot, | |
| 1077 } | |
| 1078 code.interact(prompt, None, local_vars) | |
| 1079 | |
| 1080 | |
| 1081 @need_buildbot | |
| 1082 def CMDidle(parser, args): | |
| 1083 """Lists idle slaves.""" | |
| 1084 return find_idle_busy_slaves(parser, args, True) | |
| 1085 | |
| 1086 | |
| 1087 @need_buildbot | |
| 1088 def CMDbusy(parser, args): | |
| 1089 """Lists idle slaves.""" | |
| 1090 return find_idle_busy_slaves(parser, args, False) | |
| 1091 | |
| 1092 | |
| 1093 @need_buildbot | |
| 1094 def CMDdisconnected(parser, args): | |
| 1095 """Lists disconnected slaves.""" | |
| 1096 _, args, buildbot = parser.parse_args(args) | |
| 1097 if args: | |
| 1098 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1099 for slave in buildbot.slaves: | |
| 1100 if not slave.connected: | |
| 1101 print slave.name | |
| 1102 return 0 | |
| 1103 | |
| 1104 | |
| 1105 def find_idle_busy_slaves(parser, args, show_idle): | |
| 1106 parser.add_option( | |
| 1107 '-b', '--builder', dest='builders', action='append', default=[], | |
| 1108 help='Builders to filter on') | |
| 1109 parser.add_option( | |
| 1110 '-s', '--slave', dest='slaves', action='append', default=[], | |
| 1111 help='Slaves to filter on') | |
| 1112 options, args, buildbot = parser.parse_args(args) | |
| 1113 if args: | |
| 1114 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1115 if not options.builders: | |
| 1116 options.builders = buildbot.builders.keys | |
| 1117 for builder in options.builders: | |
| 1118 builder = buildbot.builders[builder] | |
| 1119 if options.slaves: | |
| 1120 # Only the subset of slaves connected to the builder. | |
| 1121 slaves = list(set(options.slaves).intersection(set(builder.slaves.names))) | |
| 1122 if not slaves: | |
| 1123 continue | |
| 1124 else: | |
| 1125 slaves = builder.slaves.names | |
| 1126 busy_slaves = [build.slave.name for build in builder.current_builds] | |
| 1127 if show_idle: | |
| 1128 slaves = natsorted(set(slaves) - set(busy_slaves)) | |
| 1129 else: | |
| 1130 slaves = natsorted(set(slaves) & set(busy_slaves)) | |
| 1131 if options.quiet: | |
| 1132 for slave in slaves: | |
| 1133 print slave | |
| 1134 else: | |
| 1135 if slaves: | |
| 1136 print 'Builder %s: %s' % (builder.name, ', '.join(slaves)) | |
| 1137 return 0 | |
| 1138 | |
| 1139 | |
| 1140 def last_failure( | |
| 1141 buildbot, builders=None, slaves=None, steps=None, no_cache=False): | |
| 1142 """Generator returning Build object that were the last failure with the | |
| 1143 specific filters. | |
| 1144 """ | |
| 1145 builders = builders or buildbot.builders.keys | |
| 1146 for builder in builders: | |
| 1147 builder = buildbot.builders[builder] | |
| 1148 if slaves: | |
| 1149 # Only the subset of slaves connected to the builder. | |
| 1150 builder_slaves = list(set(slaves).intersection(set(builder.slaves.names))) | |
| 1151 if not builder_slaves: | |
| 1152 continue | |
| 1153 else: | |
| 1154 builder_slaves = builder.slaves.names | |
| 1155 | |
| 1156 if not no_cache and len(builder.slaves) > 2: | |
| 1157 # Unless you just want the last few builds, it's often faster to | |
| 1158 # fetch the whole thing at once, at the cost of a small hickup on | |
| 1159 # the buildbot. | |
| 1160 # TODO(maruel): Cache only N last builds or all builds since | |
| 1161 # datetime. | |
| 1162 builder.builds.cache() | |
| 1163 | |
| 1164 found = [] | |
| 1165 for build in builder.builds: | |
| 1166 if build.slave.name not in builder_slaves or build.slave.name in found: | |
| 1167 continue | |
| 1168 # Only add the slave for the first completed build but still look for | |
| 1169 # incomplete builds. | |
| 1170 if build.completed: | |
| 1171 found.append(build.slave.name) | |
| 1172 | |
| 1173 if steps: | |
| 1174 if any(build.steps[step].simplified_result is False for step in steps): | |
| 1175 yield build | |
| 1176 elif build.simplified_result is False: | |
| 1177 yield build | |
| 1178 | |
| 1179 if len(found) == len(builder_slaves): | |
| 1180 # Found all the slaves, quit. | |
| 1181 break | |
| 1182 | |
| 1183 | |
| 1184 @need_buildbot | |
| 1185 def CMDlast_failure(parser, args): | |
| 1186 """Lists all slaves that failed on that step on their last build. | |
| 1187 | |
| 1188 Example: to find all slaves where their last build was a compile failure, | |
| 1189 run with --step compile""" | |
| 1190 parser.add_option( | |
| 1191 '-S', '--step', dest='steps', action='append', default=[], | |
| 1192 help='List all slaves that failed on that step on their last build') | |
| 1193 parser.add_option( | |
| 1194 '-b', '--builder', dest='builders', action='append', default=[], | |
| 1195 help='Builders to filter on') | |
| 1196 parser.add_option( | |
| 1197 '-s', '--slave', dest='slaves', action='append', default=[], | |
| 1198 help='Slaves to filter on') | |
| 1199 parser.add_option( | |
| 1200 '-n', '--no_cache', action='store_true', | |
| 1201 help='Don\'t load all builds at once') | |
| 1202 options, args, buildbot = parser.parse_args(args) | |
| 1203 if args: | |
| 1204 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1205 print_builders = not options.quiet and len(options.builders) != 1 | |
| 1206 last_builder = None | |
| 1207 for build in last_failure( | |
| 1208 buildbot, builders=options.builders, | |
| 1209 slaves=options.slaves, steps=options.steps, | |
| 1210 no_cache=options.no_cache): | |
| 1211 | |
| 1212 if print_builders and last_builder != build.builder: | |
| 1213 print build.builder.name | |
| 1214 last_builder = build.builder | |
| 1215 | |
| 1216 if options.quiet: | |
| 1217 if options.slaves: | |
| 1218 print '%s: %s' % (build.builder.name, build.slave.name) | |
| 1219 else: | |
| 1220 print build.slave.name | |
| 1221 else: | |
| 1222 out = '%d on %s: blame:%s' % ( | |
| 1223 build.number, build.slave.name, ', '.join(build.blame)) | |
| 1224 if print_builders: | |
| 1225 out = ' ' + out | |
| 1226 print out | |
| 1227 | |
| 1228 if len(options.steps) != 1: | |
| 1229 for step in build.steps: | |
| 1230 if step.simplified_result is False: | |
| 1231 # Assume the first line is the text name anyway. | |
| 1232 summary = ', '.join(step.data['text'][1:])[:40] | |
| 1233 out = ' %s: "%s"' % (step.data['name'], summary) | |
| 1234 if print_builders: | |
| 1235 out = ' ' + out | |
| 1236 print out | |
| 1237 return 0 | |
| 1238 | |
| 1239 | |
| 1240 @need_buildbot | |
| 1241 def CMDcurrent(parser, args): | |
| 1242 """Lists current jobs.""" | |
| 1243 parser.add_option( | |
| 1244 '-b', '--builder', dest='builders', action='append', default=[], | |
| 1245 help='Builders to filter on') | |
| 1246 parser.add_option( | |
| 1247 '--blame', action='store_true', help='Only print the blame list') | |
| 1248 options, args, buildbot = parser.parse_args(args) | |
| 1249 if args: | |
| 1250 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1251 if not options.builders: | |
| 1252 options.builders = buildbot.builders.keys | |
| 1253 | |
| 1254 if options.blame: | |
| 1255 blame = set() | |
| 1256 for builder in options.builders: | |
| 1257 for build in buildbot.builders[builder].current_builds: | |
| 1258 if build.blame: | |
| 1259 for blamed in build.blame: | |
| 1260 blame.add(blamed) | |
| 1261 print '\n'.join(blame) | |
| 1262 return 0 | |
| 1263 | |
| 1264 for builder in options.builders: | |
| 1265 builder = buildbot.builders[builder] | |
| 1266 if not options.quiet and builder.current_builds: | |
| 1267 print builder.name | |
| 1268 for build in builder.current_builds: | |
| 1269 if options.quiet: | |
| 1270 print build.slave.name | |
| 1271 else: | |
| 1272 out = '%4d: slave=%10s' % (build.number, build.slave.name) | |
| 1273 out += ' duration=%5d' % (build.duration or 0) | |
| 1274 if build.eta: | |
| 1275 out += ' eta=%5.0f' % build.eta | |
| 1276 else: | |
| 1277 out += ' ' | |
| 1278 if build.blame: | |
| 1279 out += ' blame=' + ', '.join(build.blame) | |
| 1280 print out | |
| 1281 | |
| 1282 return 0 | |
| 1283 | |
| 1284 | |
| 1285 @need_buildbot | |
| 1286 def CMDbuilds(parser, args): | |
| 1287 """Lists all builds. | |
| 1288 | |
| 1289 Example: to find all builds on a single slave, run with -b bar -s foo | |
| 1290 """ | |
| 1291 parser.add_option( | |
| 1292 '-r', '--result', type='int', help='Build result to filter on') | |
| 1293 parser.add_option( | |
| 1294 '-b', '--builder', dest='builders', action='append', default=[], | |
| 1295 help='Builders to filter on') | |
| 1296 parser.add_option( | |
| 1297 '-s', '--slave', dest='slaves', action='append', default=[], | |
| 1298 help='Slaves to filter on') | |
| 1299 parser.add_option( | |
| 1300 '-n', '--no_cache', action='store_true', | |
| 1301 help='Don\'t load all builds at once') | |
| 1302 options, args, buildbot = parser.parse_args(args) | |
| 1303 if args: | |
| 1304 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1305 builders = options.builders or buildbot.builders.keys | |
| 1306 for builder in builders: | |
| 1307 builder = buildbot.builders[builder] | |
| 1308 for build in builder.builds: | |
| 1309 if not options.slaves or build.slave.name in options.slaves: | |
| 1310 if options.quiet: | |
| 1311 out = '' | |
| 1312 if options.builders: | |
| 1313 out += '%s/' % builder.name | |
| 1314 if len(options.slaves) != 1: | |
| 1315 out += '%s/' % build.slave.name | |
| 1316 out += '%d revision:%s result:%s blame:%s' % ( | |
| 1317 build.number, build.revision, build.result, ','.join(build.blame)) | |
| 1318 print out | |
| 1319 else: | |
| 1320 print build | |
| 1321 return 0 | |
| 1322 | |
| 1323 | |
| 1324 @need_buildbot | |
| 1325 def CMDcount(parser, args): | |
| 1326 """Count the number of builds that occured during a specific period. | |
| 1327 """ | |
| 1328 parser.add_option( | |
| 1329 '-o', '--over', type='int', help='Number of seconds to look for') | |
| 1330 parser.add_option( | |
| 1331 '-b', '--builder', dest='builders', action='append', default=[], | |
| 1332 help='Builders to filter on') | |
| 1333 options, args, buildbot = parser.parse_args(args) | |
| 1334 if args: | |
| 1335 parser.error('Unrecognized parameters: %s' % ' '.join(args)) | |
| 1336 if not options.over: | |
| 1337 parser.error( | |
| 1338 'Specify the number of seconds, e.g. --over 86400 for the last 24 ' | |
| 1339 'hours') | |
| 1340 builders = options.builders or buildbot.builders.keys | |
| 1341 counts = {} | |
| 1342 since = time.time() - options.over | |
| 1343 for builder in builders: | |
| 1344 builder = buildbot.builders[builder] | |
| 1345 counts[builder.name] = 0 | |
| 1346 if not options.quiet: | |
| 1347 print builder.name | |
| 1348 for build in builder.builds.iterall(): | |
| 1349 try: | |
| 1350 start_time = build.start_time | |
| 1351 except urllib2.HTTPError: | |
| 1352 # The build was probably trimmed. | |
| 1353 print >> sys.stderr, ( | |
| 1354 'Failed to fetch build %s/%d' % (builder.name, build.number)) | |
| 1355 continue | |
| 1356 if start_time >= since: | |
| 1357 counts[builder.name] += 1 | |
| 1358 else: | |
| 1359 break | |
| 1360 if not options.quiet: | |
| 1361 print '.. %d' % counts[builder.name] | |
| 1362 | |
| 1363 align_name = max(len(b) for b in counts) | |
| 1364 align_number = max(len(str(c)) for c in counts.itervalues()) | |
| 1365 for builder in sorted(counts): | |
| 1366 print '%*s: %*d' % (align_name, builder, align_number, counts[builder]) | |
| 1367 print 'Total: %d' % sum(counts.itervalues()) | |
| 1368 return 0 | |
| 1369 | |
| 1370 | |
| 1371 class OptionParser(optparse.OptionParser): | |
| 1372 def parse_args(self, args=None, values=None): | |
| 1373 """Adds common parsing.""" | |
| 1374 options, args = optparse.OptionParser.parse_args(self, args, values) | |
| 1375 levels = (logging.WARNING, logging.INFO, logging.DEBUG) | |
| 1376 logging.basicConfig(level=levels[min(options.verbose, len(levels)-1)]) | |
| 1377 return options, args | |
| 1378 | |
| 1379 def format_description(self, _): | |
| 1380 """Removes description formatting.""" | |
| 1381 return self.description | |
| 1382 | |
| 1383 | |
| 1384 def gen_parser(): | |
| 1385 """Returns an OptionParser instance with default options. | |
| 1386 | |
| 1387 It should be then processed with gen_usage() before being used. | |
| 1388 """ | |
| 1389 parser = OptionParser(version=__version__) | |
| 1390 parser.add_option( | |
| 1391 '-v', '--verbose', action='count', default=0, | |
| 1392 help='Use multiple times to increase logging leve') | |
| 1393 parser.add_option( | |
| 1394 '-q', '--quiet', action='store_true', | |
| 1395 help='Reduces the output to be parsed by scripts, independent of -v') | |
| 1396 parser.add_option( | |
| 1397 '--throttle', type='float', | |
| 1398 help='Minimum delay to sleep between requests') | |
| 1399 return parser | |
| 1400 | |
| 1401 | |
| 1402 ############################################################################### | |
| 1403 ## Generic subcommand handling code | |
| 1404 | |
| 1405 | |
| 1406 def Command(name): | |
| 1407 return getattr(sys.modules[__name__], 'CMD' + name, None) | |
| 1408 | |
| 1409 | |
| 1410 @usage('<command>') | |
| 1411 def CMDhelp(parser, args): | |
| 1412 """Print list of commands or use 'help <command>'.""" | |
| 1413 _, args = parser.parse_args(args) | |
| 1414 if len(args) == 1: | |
| 1415 return main(args + ['--help']) | |
| 1416 parser.print_help() | |
| 1417 return 0 | |
| 1418 | |
| 1419 | |
| 1420 def gen_usage(parser, command): | |
| 1421 """Modifies an OptionParser object with the command's documentation. | |
| 1422 | |
| 1423 The documentation is taken from the function's docstring. | |
| 1424 """ | |
| 1425 obj = Command(command) | |
| 1426 more = getattr(obj, 'func_usage_more') | |
| 1427 # OptParser.description prefer nicely non-formatted strings. | |
| 1428 parser.description = obj.__doc__ + '\n' | |
| 1429 parser.set_usage('usage: %%prog %s %s' % (command, more)) | |
| 1430 | |
| 1431 | |
| 1432 def main(args=None): | |
| 1433 # Do it late so all commands are listed. | |
| 1434 # pylint: disable=E1101 | |
| 1435 CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join( | |
| 1436 ' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0]) | |
| 1437 for fn in dir(sys.modules[__name__]) if fn.startswith('CMD')) | |
| 1438 | |
| 1439 parser = gen_parser() | |
| 1440 if args is None: | |
| 1441 args = sys.argv[1:] | |
| 1442 if args: | |
| 1443 command = Command(args[0]) | |
| 1444 if command: | |
| 1445 # "fix" the usage and the description now that we know the subcommand. | |
| 1446 gen_usage(parser, args[0]) | |
| 1447 return command(parser, args[1:]) | |
| 1448 | |
| 1449 # Not a known command. Default to help. | |
| 1450 gen_usage(parser, 'help') | |
| 1451 return CMDhelp(parser, args) | |
| 1452 | |
| 1453 | |
| 1454 if __name__ == '__main__': | |
| 1455 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 135363007:
Delete public commit queue to avoid confusion after move to internal repo (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/