Utilize buildCache in a smarter way to prevent pathological disk I/O storms.

third_party/buildbot_8_4p1/buildbot/status/builder.py

Index: third_party/buildbot_8_4p1/buildbot/status/builder.py
diff --git a/third_party/buildbot_8_4p1/buildbot/status/builder.py b/third_party/buildbot_8_4p1/buildbot/status/builder.py
index 3f42eb6f7e445be7020d9cd79108f854d7abc97c..a9685c14db45f4e8490557896996d50a9b9227f0 100644
--- a/third_party/buildbot_8_4p1/buildbot/status/builder.py
+++ b/third_party/buildbot_8_4p1/buildbot/status/builder.py
@@ -17,6 +17,7 @@
 import weakref
 import gc
 import os, re, itertools
+import random
 from cPickle import load, dump
 
 from zope.interface import implements
@@ -323,17 +324,58 @@ class BuilderStatus(styles.Versioned):
     def getCategory(self):
         return self.category
 
-    def getBuild(self, number):
+    def _resolveBuildNumber(self, number):
         if number < 0:
             number = self.nextBuildNumber + number
         if number < 0 or number >= self.nextBuildNumber:
             return None
+        return number
 
+    def _safeGetBuild(self, number):

[Isaac (away), 2013/07/02 02:38:25]

maybe name this var 'build_number'

[Mike Stip (use stip instead), 2013/07/02 05:43:53]

Done.

         try:
             return self.getBuildByNumber(number)
         except IndexError:
             return None
 
+    def getBuild(self, number):
+        number = self._resolveBuildNumber(number)
+
+        if number is None:
+            return None
+
+        return self._safeGetBuild(number)
+
+    def getBuilds(self, numbers):
+        """Cache-aware method to get multiple builds.
+
+        Prevents cascading evict/load when multiple builds are requested in
+        succession: requesting build 1 evicts build 2, requesting build 2 evicts
+        build 3, etc.
+
+        We query the buildCache and load hits first, then misses.  When loading,
+        we randomize the load order to alleviate the problem when external web
+        requests load builds sequentially (they don't have access to this
+        function).
+        """
+
+        numbers = list(enumerate(self._resolveBuildNumber(x) for x in numbers))
+        random.shuffle(numbers)
+
+        builds = [None] * len(numbers)
+        misses = []
+        for idx, number in numbers:

[Isaac (away), 2013/07/02 02:38:25]

better variable name for number, perhaps build_number?

[Mike Stip (use stip instead), 2013/07/02 05:43:53]

Done.

+            if number is None:
+                continue
+            if number in self.buildCache.cache:
+                builds[idx] = self._safeGetBuild(number)
+            else:
+                misses.append((idx, number))
+
+        for idx, number in misses:
+            builds[idx] = self._safeGetBuild(number)
+
+        return builds
+
     def getEvent(self, number):
         try:
             return self.events[number]
@@ -616,6 +658,11 @@ class BuilderStatus(styles.Versioned):
         # Collect build numbers.
         # Important: Only grab the *cached* builds numbers to reduce I/O.
         current_builds = [b.getNumber() for b in self.currentBuilds]
+
+        # Populates buildCache with last N builds.
+        buildnums = range(-1, -(self.buildCacheSize - 1), -1)
+        self.getBuilds(buildnums)
+
         cached_builds = list(set(self.buildCache.cache.keys() + current_builds))
         cached_builds.sort()
         result['cachedBuilds'] = cached_builds