rebaseline_server: periodically log tasks_queue size

gm/rebaseline_server/imagediffdb.py

 #!/usr/bin/python
 
 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Calulate differences between image pairs, and store them in a database.
 """
@@ skipping 37 matching lines @@
 # NOTE: Keep these in sync with static/constants.js
 KEY__DIFFERENCES__MAX_DIFF_PER_CHANNEL = 'maxDiffPerChannel'
 KEY__DIFFERENCES__NUM_DIFF_PIXELS = 'numDifferingPixels'
 KEY__DIFFERENCES__PERCENT_DIFF_PIXELS = 'percentDifferingPixels'
 KEY__DIFFERENCES__PERCEPTUAL_DIFF = 'perceptualDifference'
 
 # Special values within ImageDiffDB._diff_dict
 _DIFFRECORD_FAILED = 'failed'
 _DIFFRECORD_PENDING = 'pending'
 
+# How often to report tasks_queue size
+QUEUE_LOGGING_GRANULARITY = 1000
+
 # Temporary variable to keep track of how many times we download
 # the same file in multiple threads.
 # TODO(epoger): Delete this, once we see that the number stays close to 0.
 global_file_collisions = 0
 
 
 class DiffRecord(object):
   """ Record of differences between two images. """
 
   def __init__(self, gs, storage_root,
@@ skipping 165 matching lines @@
     """
     Args:
       storage_root: string; root path within the DB will store all of its stuff
       gs: instance of GSUtils object we can use to download images
       num_worker_threads: how many threads that download images and
           generate diffs simultaneously
     """
     self._storage_root = storage_root
     self._gs = gs
 
+    # Mechanism for reporting queue size periodically.
+    self._last_queue_size_reported = None
+    self._queue_size_report_lock = threading.RLock()
+
     # Dictionary of DiffRecords, keyed by (expected_image_locator,
     # actual_image_locator) tuples.
     # Values can also be _DIFFRECORD_PENDING, _DIFFRECORD_FAILED.
     #
     # Any thread that modifies _diff_dict must first acquire
     # _diff_dict_writelock!
     #
     # TODO(epoger): Disk is limitless, but RAM is not... so, we should probably
     # remove items from self._diff_dict if they haven't been accessed for a
     # long time.  We can always regenerate them by diffing the images we
@@ skipping 10 matching lines @@
     # The queue maxsize must be 0 (infinite size queue), so that asynchronous
     # calls can return as soon as possible.
     self._tasks_queue = Queue.Queue(maxsize=0)
     self._workers = []
     for i in range(num_worker_threads):
       worker = threading.Thread(target=self.worker, args=(i,))
       worker.daemon = True
       worker.start()
       self._workers.append(worker)
 
+  def log_queue_size_if_changed(self, limit_verbosity=True):
+    """Log the size of self._tasks_queue, if it has changed since the last call.
+
+    Reports the current queue size, using log.info(), unless the queue is the
+    same size as the last time we reported it.
+
+    Args:
+      limit_verbosity: if True, only log if the queue size is a multiple of
+          QUEUE_LOGGING_GRANULARITY
+    """
+    self._queue_size_report_lock.acquire()
+    try:
+      size = self._tasks_queue.qsize()
+      if size == self._last_queue_size_reported:
+        return
+      if limit_verbosity and (size % QUEUE_LOGGING_GRANULARITY != 0):
+        return
+      logging.info('tasks_queue size is %d' % size)
+      self._last_queue_size_reported = size
+    finally:
+      self._queue_size_report_lock.release()
+

[stephana, 2014/08/11 13:35:18]

This is a nit, but why do you need a lock for reading the queue size ?

[epoger, 2014/08/11 18:49:09]

Actually, we need it for self._last_queue_size_reported; without it, several
threads can report at the same queue size.  Added a comment line to hopefully
make that clearer.

   def worker(self, worker_num):
     """Launch a worker thread that pulls tasks off self._tasks_queue.
 
     Args:
       worker_num: (integer) which worker this is
     """
     while True:
+      self.log_queue_size_if_changed()
       params = self._tasks_queue.get()
       key, expected_image_url, actual_image_url = params
       try:
         diff_record = DiffRecord(
             self._gs, self._storage_root,
             expected_image_url=expected_image_url,
             expected_image_locator=key[0],
             actual_image_url=actual_image_url,
             actual_image_locator=key[1])
       except Exception:
@@ skipping 46 matching lines @@
       if not key in self._diff_dict:
         # If we have already requested a diff between these two images,
         # we don't need to request it again.
         must_add_to_queue = True
         self._diff_dict[key] = _DIFFRECORD_PENDING
     finally:
       self._diff_dict_writelock.release()
 
     if must_add_to_queue:
       self._tasks_queue.put((key, expected_image_url, actual_image_url))
+      self.log_queue_size_if_changed()
 
   def get_diff_record(self, expected_image_locator, actual_image_locator):
     """Returns the DiffRecord for this image pair.
 
     This call will block until the diff record is available, or we were unable
     to generate it.
 
     Args:
       expected_image_locator: a unique ID string under which we will store the
           expected image within storage_root (probably including a checksum to
@@ skipping 101 matching lines @@
 
   Args:
     expected_image_locator: locator string pointing at expected image
     actual_image_locator: locator string pointing at actual image
 
   Returns: already-sanitized locator where the diffs between expected and
       actual images can be found
   """
   return "%s-vs-%s" % (_sanitize_locator(expected_image_locator),
                        _sanitize_locator(actual_image_locator))