Resident set size (RSS) of multiple processes without double-counts on Linux.

tools/linux/procfs.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # A Python library to read and store procfs (/proc) information on Linux.
 #
 # Each information storage class in this file stores original data as original
 # as reasonablly possible. Translation is done when requested. It is to make it
 # always possible to probe the original data.
 
 
 import collections
 import logging
 import os
 import re
 import struct
 import sys
 
 
-LOGGER = logging.getLogger('procfs')
+class _NullHandler(logging.Handler):
+  def emit(self, record):
+    pass
+
+
+_LOGGER = logging.getLogger('procfs')
+_LOGGER.addHandler(_NullHandler())
 
 
 class ProcStat(object):
   """Reads and stores information in /proc/pid/stat."""
   _PATTERN = re.compile(r'^'
                         '(?P<PID>-?[0-9]+) '
                         '\((?P<COMM>.+)\) '
                         '(?P<STATE>[RSDZTW]) '
                         '(?P<PPID>-?[0-9]+) '
                         '(?P<PGRP>-?[0-9]+) '
@@ skipping 508 matching lines @@
     self._vma_internals = vma_internals
     self._in_process_dup = in_process_dup
 
   @staticmethod
   def load(pid, maps):
     total_present = 0
     total_swapped = 0
     total_vsize = 0
     in_process_dup = 0
     vma_internals = collections.OrderedDict()
+    process_pageframe_set = set()
 
     pagemap_fd = os.open(
         os.path.join('/proc', str(pid), 'pagemap'), os.O_RDONLY)
     for vma in maps:
       present = 0
       swapped = 0
       vsize = 0
       pageframes = collections.defaultdict(int)
-      pageframes_set = set()
       begin_offset = ProcPagemap._offset(vma.begin)
       chunk_size = ProcPagemap._offset(vma.end) - begin_offset
       os.lseek(pagemap_fd, begin_offset, os.SEEK_SET)
       buf = os.read(pagemap_fd, chunk_size)
       if len(buf) < chunk_size:
-        LOGGER.warn('Failed to read pagemap at 0x%x.' % vma.begin)
+        _LOGGER.warn('Failed to read pagemap at 0x%x in %d.' % (vma.begin, pid))
       pagemap_values = struct.unpack(
           '=%dQ' % (len(buf) / ProcPagemap._BYTES_PER_PAGEMAP_VALUE), buf)
       for pagemap_value in pagemap_values:
         vsize += ProcPagemap._BYTES_PER_OS_PAGE
         if pagemap_value & ProcPagemap._MASK_PRESENT:
-          if (pagemap_value & ProcPagemap._MASK_PFN) in pageframes_set:
+          if (pagemap_value & ProcPagemap._MASK_PFN) in process_pageframe_set:
             in_process_dup += ProcPagemap._BYTES_PER_OS_PAGE
           else:
-            pageframes_set.add(pagemap_value & ProcPagemap._MASK_PFN)
+            process_pageframe_set.add(pagemap_value & ProcPagemap._MASK_PFN)
           if (pagemap_value & ProcPagemap._MASK_PFN) not in pageframes:
             present += ProcPagemap._BYTES_PER_OS_PAGE
           pageframes[pagemap_value & ProcPagemap._MASK_PFN] += 1
         if pagemap_value & ProcPagemap._MASK_SWAPPED:
           swapped += ProcPagemap._BYTES_PER_OS_PAGE
       vma_internals[vma] = ProcPagemap.VMA(vsize, present, swapped, pageframes)
       total_present += present
       total_swapped += swapped
       total_vsize += vsize
     os.close(pagemap_fd)
@@ skipping 89 matching lines @@
   def statm(self):
     return self._statm
 
   @property
   def status(self):
     return self._status
 
 
 def main(argv):
   """The main function for manual testing."""
-
-  LOGGER.setLevel(logging.DEBUG)
+  _LOGGER.setLevel(logging.WARNING)
   handler = logging.StreamHandler()
-  handler.setLevel(logging.INFO)
-  formatter = logging.Formatter('%(message)s')
-  handler.setFormatter(formatter)
-  LOGGER.addHandler(handler)
+  handler.setLevel(logging.WARNING)
+  handler.setFormatter(logging.Formatter(
+      '%(asctime)s:%(name)s:%(levelname)s:%(message)s'))
+  _LOGGER.addHandler(handler)
 
   pids = []
   for arg in argv[1:]:
     try:
       pid = int(arg)
     except ValueError:
       raise SyntaxError("%s is not an integer." % arg)
     else:
       pids.append(pid)
 
@@ skipping 15 matching lines @@
     print ' status: %d (Peak:%d)' % (procs[pid].status.vm_rss * 1024,
                                      procs[pid].status.vm_hwm * 1024)
     print '  smaps: %d' % (procs[pid].smaps.rss * 1024)
     print 'pagemap: %d' % procs[pid].pagemap.present
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))