Switch to the good LRU implementation.

third_party/buildbot_8_4p1/buildbot/status/builder.py

 # This file is part of Buildbot.  Buildbot is free software: you can
 # redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation, version 2.
 #
 # This program is distributed in the hope that it will be useful, but WITHOUT
 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc., 51
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
 
 import weakref
 import gc
 import os, re, itertools
 from cPickle import load, dump
 
 from zope.interface import implements
 from twisted.python import log, runtime
 from twisted.persisted import styles
 from buildbot.process import metrics
 from buildbot import interfaces, util
+from buildbot.util.lru import SyncLRUCache
 from buildbot.status.event import Event
 from buildbot.status.build import BuildStatus
 from buildbot.status.buildrequest import BuildRequestStatus
 
 # user modules expect these symbols to be present here
 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED
 from buildbot.status.results import EXCEPTION, RETRY, Results, worst_status
 _hush_pyflakes = [ SUCCESS, WARNINGS, FAILURE, SKIPPED,
                    EXCEPTION, RETRY, Results, worst_status ]
 
@@ skipping 42 matching lines @@
         self.slavenames = []
         self.events = []
         # these three hold Events, and are used to retrieve the current
         # state of the boxes.
         self.lastBuildStatus = None
         #self.currentBig = None
         #self.currentSmall = None
         self.currentBuilds = []
         self.nextBuild = None
         self.watchers = []
-        self.buildCache = weakref.WeakValueDictionary()
-        self.buildCache_LRU = []
+        self.buildCache = SyncLRUCache(self.cacheMiss, self.buildCacheSize)
         self.logCompressionLimit = False # default to no compression for tests
         self.logCompressionMethod = "bz2"
         self.logMaxSize = None # No default limit
         self.logMaxTailSize = None # No tail buffering
 
     # persistence
 
     def __getstate__(self):
         # when saving, don't record transient stuff like what builds are
         # currently running, because they won't be there when we start back
         # up. Nor do we save self.watchers, nor anything that gets set by our
         # parent like .basedir and .status
         d = styles.Versioned.__getstate__(self)
         d['watchers'] = []
         del d['buildCache']
-        del d['buildCache_LRU']
         for b in self.currentBuilds:
             b.saveYourself()
             # TODO: push a 'hey, build was interrupted' event
         del d['currentBuilds']
         d.pop('pendingBuilds', None)
         del d['currentBigState']
         del d['basedir']
         del d['status']
         del d['nextBuildNumber']
         return d
 
     def __setstate__(self, d):
         # when loading, re-initialize the transient stuff. Remember that
         # upgradeToVersion1 and such will be called after this finishes.
         styles.Versioned.__setstate__(self, d)
-        self.buildCache = weakref.WeakValueDictionary()
-        self.buildCache_LRU = []
+        self.buildCache = SyncLRUCache(self.cacheMiss, self.buildCacheSize)
         self.currentBuilds = []
         self.watchers = []
         self.slavenames = []
         # self.basedir must be filled in by our parent
         # self.status must be filled in by our parent
 
     def reconfigFromBuildmaster(self, buildmaster):
         # Note that we do not hang onto the buildmaster, since this object
         # gets pickled and unpickled.
         if buildmaster.buildCacheSize is not None:
             self.buildCacheSize = buildmaster.buildCacheSize
+            self.buildCache.set_max_size(buildmaster.buildCacheSize)
 
     def upgradeToVersion1(self):
         if hasattr(self, 'slavename'):
             self.slavenames = [self.slavename]
             del self.slavename
         if hasattr(self, 'nextBuildNumber'):
             del self.nextBuildNumber # determineNextBuildNumber chooses this
         self.wasUpgraded = True
 
     def determineNextBuildNumber(self):
@@ skipping 34 matching lines @@
         try:
             dump(self, open(tmpfilename, "wb"), -1)
             if runtime.platformType  == 'win32':
                 # windows cannot rename a file on top of an existing one
                 if os.path.exists(filename):
                     os.unlink(filename)
             os.rename(tmpfilename, filename)
         except:
             log.msg("unable to save builder %s" % self.name)
             log.err()
-        
+
 
     # build cache management
 
     def makeBuildFilename(self, number):
         return os.path.join(self.basedir, "%d" % number)
 
-    def touchBuildCache(self, build):
-        self.buildCache[build.number] = build
-        if build in self.buildCache_LRU:
-            self.buildCache_LRU.remove(build)
-        self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ]
-        return build
+    def getBuildByNumber(self, number):
+        return self.buildCache.get(number)
 
-    def getBuildByNumber(self, number):
-        # first look in currentBuilds
-        for b in self.currentBuilds:
-            if b.number == number:
-                return self.touchBuildCache(b)
-
-        # then in the buildCache
-        if number in self.buildCache:
-            metrics.MetricCountEvent.log("buildCache.hits", 1)
-            return self.touchBuildCache(self.buildCache[number])
-        metrics.MetricCountEvent.log("buildCache.misses", 1)
-
-        # then fall back to loading it from disk
+    def loadBuildFromFile(self, number):
         filename = self.makeBuildFilename(number)
         try:
             log.msg("Loading builder %s's build %d from on-disk pickle"
                 % (self.name, number))
             build = load(open(filename, "rb"))
             build.builder = self
 
             # (bug #1068) if we need to upgrade, we probably need to rewrite
             # this pickle, too.  We determine this by looking at the list of
             # Versioned objects that have been unpickled, and (after doUpgrade)
             # checking to see if any of them set wasUpgraded.  The Versioneds'
             # upgradeToVersionNN methods all set this.
             versioneds = styles.versionedsToUpgrade
             styles.doUpgrade()
             if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]:
                 log.msg("re-writing upgraded build pickle")
                 build.saveYourself()
 
             # handle LogFiles from after 0.5.0 and before 0.6.5
             build.upgradeLogfiles()
             # check that logfiles exist
             build.checkLogfiles()
-            return self.touchBuildCache(build)
+            return build
         except IOError:
             raise IndexError("no such build %d" % number)
         except EOFError:
             raise IndexError("corrupted build pickle %d" % number)
 
+    def cacheMiss(self, number):
+        # first look in currentBuilds
+        for b in self.currentBuilds:
+            if b.number == number:
+              return b
+        # then fall back to loading it from disk
+        return self.loadBuildFromFile(number)
+
     def prune(self, events_only=False):
         # begin by pruning our own events
         self.events = self.events[-self.eventHorizon:]
 
         if events_only:
             return
 
         gc.collect()
 
         # get the horizons straight
@@ skipping 26 matching lines @@
             is_logfile = False
             if mo:
                 num = int(mo.group(1))
             else:
                 mo = build_log_re.match(filename)
                 if mo:
                     num = int(mo.group(1))
                     is_logfile = True
 
             if num is None: continue
-            if num in self.buildCache: continue
+            if num in self.buildCache.cache: continue
 
             if (is_logfile and num < earliest_log) or num < earliest_build:
                 pathname = os.path.join(self.basedir, filename)
                 log.msg("pruning '%s'" % pathname)
                 try: os.unlink(pathname)
                 except OSError: pass
 
     # IBuilderStatus methods
     def getName(self):
         return self.name
@@ skipping 202 matching lines @@
 
     # buildStarted is called by our child BuildStatus instances
     def buildStarted(self, s):
         """Now the BuildStatus object is ready to go (it knows all of its
         Steps, its ETA, etc), so it is safe to notify our watchers."""
 
         assert s.builder is self # paranoia
         assert s.number == self.nextBuildNumber - 1
         assert s not in self.currentBuilds
         self.currentBuilds.append(s)
-        self.touchBuildCache(s)
+        self.buildCache.put(s.number, s)
 
         # now that the BuildStatus is prepared to answer queries, we can
         # announce the new build to all our watchers
 
         for w in self.watchers: # TODO: maybe do this later? callLater(0)?
             try:
                 receiver = w.buildStarted(self.getName(), s)
                 if receiver:
                     if type(receiver) == type(()):
                         s.subscribe(receiver[0], receiver[1])
@@ skipping 89 matching lines @@
         result['basedir'] = os.path.basename(self.basedir)
         result['category'] = self.category
         result['slaves'] = self.slavenames
         #result['url'] = self.parent.getURLForThing(self)
         # TODO(maruel): Add cache settings? Do we care?
 
         # Transient
         # Collect build numbers.
         # Important: Only grab the *cached* builds numbers to reduce I/O.
         current_builds = [b.getNumber() for b in self.currentBuilds]
-        cached_builds = list(set(self.buildCache.keys() + current_builds))
+        cached_builds = list(set(self.buildCache.cache.keys() + current_builds))
         cached_builds.sort()
         result['cachedBuilds'] = cached_builds
         result['currentBuilds'] = current_builds
         result['state'] = self.getState()[0]
         # lies, but we don't have synchronous access to this info; use
         # asDict_async instead
         result['pendingBuilds'] = 0
         return result
 
     def asDict_async(self):
         """Just like L{asDict}, but with a nonzero pendingBuilds."""
         result = self.asDict()
         d = self.getPendingBuildRequestStatuses()
         def combine(statuses):
             result['pendingBuilds'] = len(statuses)
             return result
         d.addCallback(combine)
         return d
 
     def getMetrics(self):
         return self.botmaster.parent.metrics
 
 # vim: set ts=4 sts=4 sw=4 et: