Limited number of buildsets that can be used to initialize RietveldPollerWithCache's cache

scripts/master/try_job_rietveld.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import hashlib
 import json
 import os
 import pytz
 import time
 import urllib
 import urlparse
 
 from buildbot.changes import base
 from buildbot.schedulers.trysched import BadJobfile
 from buildbot.status.builder import EXCEPTION
 from twisted.application import internet
 from twisted.internet import defer
 from twisted.python import log
 from twisted.web import client
 
 from master import master_utils
 from master.try_job_base import TryJobBase
 
 
+# Number of recent buildsets used to initialize RietveldPollerWithCache's cache.
+MAX_RECENT_BUILDSETS_TO_INIT_CACHE = 10000
+
+
 class _ValidUserPoller(internet.TimerService):
   """Check chromium-access for users allowed to send jobs from Rietveld.
   """
   # The name of the file that contains the password for authenticating
   # requests to chromium-access.
   _PWD_FILE = '.try_job_rietveld_password'
   _NORMAL_DOMAIN = '@chromium.org'
   _SPECIAL_DOMAIN = '@google.com'
 
   def __init__(self, interval):
@@ skipping 198 matching lines @@
     return pollDeferred
 
   def setServiceParent(self, parent):
     base.PollingChangeSource.setServiceParent(self, parent)
     self._try_job_rietveld = parent
 
   @defer.inlineCallbacks
   def _InitProcessedKeysCache(self):
     log.msg('[RPWC] Initializing processed keys cache...')
 
-    # Get all BuildBot build requests.
-    brdicts = yield self.master.db.buildrequests.getBuildRequests()
+    # Get recent BuildBot buildsets.

[Paweł Hajdan Jr., 2014/06/03 08:08:47]

nit: Add a comment here that we need to limit the number since otherwise
fetching all build requests would take days.

[Sergiy Byelozyorov, 2014/06/03 08:20:01]

Done.

+    bsdicts = yield self.master.db.buildsets.getRecentBuildsets(
+        MAX_RECENT_BUILDSETS_TO_INIT_CACHE)
 
-    log.msg('[RPWC] Received %d build request dicts' % len(brdicts))
+    log.msg('[RPWC] Received %d buildset dicts' % len(bsdicts))
 
     def asNaiveUTC(dt):
       if dt is None:
         return datetime.datetime.now()
       if dt.tzinfo is None:
         return dt
       utc_datetime = dt.astimezone(pytz.utc)
       return utc_datetime.replace(tzinfo=None)
 
     # Compose a map of buildset ids to the submission timestamp.
     buildsets = {}
-    for brdict in brdicts:
-      bsid = brdict.get('buildsetid')
+    for bsdict in bsdicts:
+      bsid = bsdict.get('bsid')
       if bsid is not None:
-        buildsets[bsid] = asNaiveUTC(brdict.get('submitted_at'))
+        buildsets[bsid] = asNaiveUTC(bsdict.get('submitted_at'))
 
     log.msg('[RPWC] Processing %d buildsets' % len(buildsets))
 
     # Find jobs for each buildset and add them to the processed keys cache.
     self._processed_keys = {}
     for bsid in buildsets.keys():
       log.msg('[RPWC] Loading properties of the buildset %d' % bsid)
       bsprops = yield self.master.db.buildsets.getBuildsetProperties(bsid)
       if 'try_job_key' in bsprops:
         key = bsprops['try_job_key'][0]
@@ skipping 220 matching lines @@
         else:
           self.processed_keys.add(job['key'])
           log.err('Rietveld not updated: no corresponding service found.')
 
   # TryJobBase overrides:
   def setServiceParent(self, parent):
     TryJobBase.setServiceParent(self, parent)
     self._poller.setServiceParent(self)
     self._poller.master = self.master
     self._valid_users.setServiceParent(self)