Histograms.xml python script housekeeping

tools/metrics/histograms/find_unmapped_histograms.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Scans the Chromium source for histograms that are absent from histograms.xml.
 
 This is a heuristic scan, so a clean run of this script does not guarantee that
 all histograms in the Chromium source are properly mapped.  Notably, field
 trials are entirely ignored by this script.
 
 """
 
-import commands
-import extract_histograms
 import hashlib
 import logging
 import optparse
 import os
 import re
+import subprocess
 import sys
 
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))  # tools/metrics
+from common import path_util
+from histograms import extract_histograms
+
 
 ADJACENT_C_STRING_REGEX = re.compile(r"""
     ("      # Opening quotation mark
     [^"]*)  # Literal string contents
     "       # Closing quotation mark
     \s*     # Any number of spaces
     "       # Another opening quotation mark
     """, re.VERBOSE)
 CONSTANT_REGEX = re.compile(r"""
     (\w*::)?  # Optional namespace
     k[A-Z]    # Match a constant identifier: 'k' followed by an uppercase letter
     \w*       # Match the rest of the constant identifier
     $         # Make sure there's only the identifier, nothing else
     """, re.VERBOSE)
 HISTOGRAM_REGEX = re.compile(r"""
     UMA_HISTOGRAM  # Match the shared prefix for standard UMA histogram macros
     \w*            # Match the rest of the macro name, e.g. '_ENUMERATION'
     \(             # Match the opening parenthesis for the macro
     \s*            # Match any whitespace -- especially, any newlines
     ([^,)]*)       # Capture the first parameter to the macro
     [,)]           # Match the comma/paren that delineates the first parameter
     """, re.VERBOSE)
 
 
+def RunGit(command):
+  """Run a git subcommand, returning its output."""
+  # On Windows, use shell=True to get PATH interpretation.
+  command = ['git'] + command
+  logging.info(' '.join(command))
+  shell = (os.name == 'nt')
+  proc = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE)
+  out = proc.communicate()[0].strip()
+  return out
+
+
 class DirectoryNotFoundException(Exception):
   """Base class to distinguish locally defined exceptions from standard ones."""
   def __init__(self, msg):
     self.msg = msg
 
   def __str__(self):
     return self.msg
 
 
-def findDefaultRoot():
-  """Find the root of the chromium repo, in case the script is run from the
-  histograms dir.
-
-  Returns:
-    string: path to the src dir of the repo.
-
-  Raises:
-    DirectoryNotFoundException if the target directory cannot be found.
-  """
-  path = os.getcwd()
-  while path:
-    head, tail = os.path.split(path)
-    if tail == 'src':
-      return path
-    if path == head:
-      break
-    path = head
-  raise DirectoryNotFoundException('Could not find src/ dir')
-
-
 def collapseAdjacentCStrings(string):
   """Collapses any adjacent C strings into a single string.
 
   Useful to re-combine strings that were split across multiple lines to satisfy
   the 80-col restriction.
 
   Args:
     string: The string to recombine, e.g. '"Foo"\n    "bar"'
 
   Returns:
@@ skipping 37 matching lines @@
                   histogram)
 
 
 def readChromiumHistograms():
   """Searches the Chromium source for all histogram names.
 
   Also prints warnings for any invocations of the UMA_HISTOGRAM_* macros with
   names that might vary during a single run of the app.
 
   Returns:
-    A set cotaining any found literal histogram names.
+    A set containing any found literal histogram names.
   """
   logging.info('Scanning Chromium source for histograms...')
 
   # Use git grep to find all invocations of the UMA_HISTOGRAM_* macros.
   # Examples:
   #   'path/to/foo.cc:420:  UMA_HISTOGRAM_COUNTS_100("FooGroup.FooName",'
   #   'path/to/bar.cc:632:  UMA_HISTOGRAM_ENUMERATION('
-  locations = commands.getoutput('git gs UMA_HISTOGRAM').split('\n')
+  locations = RunGit(['gs', 'UMA_HISTOGRAM']).split('\n')
   filenames = set([location.split(':')[0] for location in locations])
 
   histograms = set()
   for filename in filenames:
     contents = ''
     with open(filename, 'r') as f:
       contents = f.read()
 
     matches = set(HISTOGRAM_REGEX.findall(contents))
     for histogram in matches:
       histogram = collapseAdjacentCStrings(histogram)
 
       # Must begin and end with a quotation mark.
-      if histogram[0] != '"' or histogram[-1] != '"':
+      if not histogram or histogram[0] != '"' or histogram[-1] != '"':

[Ilya Sherman, 2015/05/31 00:41:02]

Why did you need to add "not histogram" to this condition?  I'm not following
what change caused the need for that.

[ncarter (slow), 2015/06/01 17:04:12]

This comment caused this script to die because we got a null match in
HISTOGRAM_REGEX (the pattern matched but there was nothing in the capture group)
https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/net...

The other way to fix this would be to change the * inside the first-parameter
capture group for HISTOGRAM_REGEX to be a +. [I considered trying to filter out
comment lines using lookaround but my regex foo isn't strong enough and it
seemed like a yak shave]

Let me know which you prefer.

[Ilya Sherman, 2015/06/01 20:25:45]

Okay, thanks for the explanation.  I was mostly just curious, and am fine with
either solution.  It sounds like Alexei would prefer to ignore comment lines
entirely.  Alexei, could you possibly help craft a regex that does this?  I'm
not sure offhand how to do that, and it sounds like Nick is also unsure.

         logNonLiteralHistogram(filename, histogram)
         continue
 
       # Must not include any quotation marks other than at the beginning or end.
       histogram_stripped = histogram.strip('"')
       if '"' in histogram_stripped:
         logNonLiteralHistogram(filename, histogram)
         continue
 
       histograms.add(histogram_stripped)
@@ skipping 19 matching lines @@
     name: The string to hash (a histogram name).
 
   Returns:
     Histogram hash as a string representing a hex number (with leading 0x).
   """
   return '0x' + hashlib.md5(name).hexdigest()[:16]
 
 
 def main():
   # Find default paths.
-  default_root = findDefaultRoot()
-  default_histograms_path = os.path.join(
-      default_root, 'tools/metrics/histograms/histograms.xml')
-  default_extra_histograms_path = os.path.join(
-      default_root, 'tools/histograms/histograms.xml')
+  default_root = path_util.GetInputFile('/')
+  default_histograms_path = path_util.GetInputFile(
+      'tools/metrics/histograms/histograms.xml')
+  default_extra_histograms_path = path_util.GetInputFile(
+      'tools/histograms/histograms.xml')
 
   # Parse command line options
   parser = optparse.OptionParser()
   parser.add_option(
     '--root-directory', dest='root_directory', default=default_root,
     help='scan within DIRECTORY for histograms [optional, defaults to "%s"]' %
         default_root,
     metavar='DIRECTORY')
   parser.add_option(
     '--histograms-file', dest='histograms_file_location',
@@ skipping 37 matching lines @@
     logging.info('Histograms in Chromium but not in XML files:')
     logging.info('-------------------------------------------------')
     for histogram in sorted(unmapped_histograms):
       logging.info('  %s - %s', histogram, hashHistogramName(histogram))
   else:
     logging.info('Success!  No unmapped histograms found.')
 
 
 if __name__ == '__main__':
   main()