Factored out calls to client.getPage to factor out common constants (timeout and buildbot name) and…

scripts/master/try_job_rietveld.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import hashlib
 import json
 import os
 import pytz
 import time
@@ skipping 204 matching lines @@
   """Polls Rietveld for any pending patch sets to build.
 
   Periodically polls Rietveld to see if any patch sets have been marked by
   users to be tried. If so, send them to the trybots. Uses cursor to download
   all pages within a single poll. To avoid sending same jobs, keeps a cache of
   the jobs that were already processed thus reducing chances of a duplicate job
   submission and increasing robustness to bugs in Rietveld. On the first poll
   this cache is initialized with jobs currently pending on the Buildbot.
   """
 
-  def __init__(self, pending_jobs_url, interval):
+  def __init__(self, pending_jobs_url, interval, timeout=2*60):
     """
     Args:
       pending_jobs_url: Rietveld URL string used to retrieve jobs to try.
       interval: Interval used to poll Rietveld, in seconds.
     """
     self.pollInterval = interval
     self._try_job_rietveld = None
     self._pending_jobs_url = pending_jobs_url
     self._processed_keys = None
+    self.timeout = timeout
+
+  def getPage(self, url):  # pylint: disable=R0201
+    """Schedules a page at `url` to be downloaded. Returns a deferred."""
+    return client.getPage(url, agent='buildbot', timeout=self.timeout)
 
   # base.PollingChangeSource overrides:
   def poll(self):
     """Polls Rietveld for any pending try jobs and submit them.
 
     Returns:
       A deferred object to be called once the operation completes.
     """
 
     # Make sure setServiceParent was called before this method is called.
     assert self._try_job_rietveld
 
     # Check if we have valid user list, otherwise - do not submit any jobs. This
     # is different from having no valid users in the list, in which case
     # SubmitJobs will correctly discard the jobs.
     if not self._try_job_rietveld.has_valid_user_list():
       log.msg('[RPWC] No valid user list. Ignoring the poll request.')
       return
 
     log.msg('[RPWC] Poll started')
     log.msg('[RPWC] Downloading %s...' % self._pending_jobs_url)
-    pollDeferred = client.getPage(self._pending_jobs_url, agent='buildbot',
-                                  timeout=2*60)
+    pollDeferred = self.getPage(self._pending_jobs_url)
     pollDeferred.addCallback(self._ProcessResults)
     pollDeferred.addErrback(log.err, '[RPWC] error')
     return pollDeferred
 
   def setServiceParent(self, parent):
     base.PollingChangeSource.setServiceParent(self, parent)
     self._try_job_rietveld = parent
 
   @defer.inlineCallbacks
   def _InitProcessedKeysCache(self):
@@ skipping 47 matching lines @@
       yield self._InitProcessedKeysCache()
 
     prev_cursor = None
     # TODO(sergiyb): Change logic to use 'more' field when it is implemented on
     # Rietveld. This field will indicate whether there are more pages.
     while len(results['jobs']) and prev_cursor != results['cursor']:
       all_jobs.extend(results['jobs'])
       next_url = self._pending_jobs_url + '&cursor=%s' % str(results['cursor'])
       prev_cursor = results['cursor']
       log.msg('[RPWC] Downloading %s...' % next_url)
-      page_json = yield client.getPage(next_url, agent='buildbot', timeout=2*60)
+      page_json = yield self.getPage(next_url)
       results = json.loads(page_json)
 
     log.msg('[RPWC] Retrieved %d jobs' % len(all_jobs))
 
     # Rietveld uses AppEngine NDB API, which serves naive UTC datetimes.
     cutoff_timestamp = datetime.datetime.utcnow() - datetime.timedelta(hours=6)
 
     log.msg('[RPWC] Cache contains %d jobs' % len(self._processed_keys))
 
     # Find new jobs and put them into cache.
@@ skipping 184 matching lines @@
         else:
           self.processed_keys.add(job['key'])
           log.err('Rietveld not updated: no corresponding service found.')
 
   # TryJobBase overrides:
   def setServiceParent(self, parent):
     TryJobBase.setServiceParent(self, parent)
     self._poller.setServiceParent(self)
     self._poller.master = self.master
     self._valid_users.setServiceParent(self)