Update UMA user actions parsing, primarily to include WebUI metrics.

chrome/tools/extract_actions.py

 #!/usr/bin/python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Extract UserMetrics "actions" strings from the Chrome source.
 
 This program generates the list of known actions we expect to see in the
 user behavior logs.  It walks the Chrome source, looking for calls to
 UserMetrics functions, extracting actions and warning on improper calls,
 as well as generating the lists of possible actions in situations where
 there are many possible actions.
 
 See also:
   chrome/browser/user_metrics.h
   http://wiki.corp.google.com/twiki/bin/view/Main/ChromeUserExperienceMetrics
 
 If run with a "--hash" argument, chromeactions.txt will be updated.
 """
 
 __author__ = 'evanm (Evan Martin)'
 
+import hashlib
+from HTMLParser import HTMLParser
 import os
 import re
 import sys
-import hashlib
 
 sys.path.insert(1, os.path.join(sys.path[0], '..', '..', 'tools', 'python'))
 from google import path_utils
 
 # Files that are known to use UserMetrics::RecordComputedAction(), which means
 # they require special handling code in this script.
 # To add a new file, add it to this list and add the appropriate logic to
 # generate the known actions to AddComputedActions() below.
 KNOWN_COMPUTED_USERS = (
   'back_forward_menu_model.cc',
   'options_page_view.cc',
   'render_view_host.cc',  # called using webkit identifiers
   'user_metrics.cc',  # method definition
   'new_tab_ui.cc',  # most visited clicks 1-9
   'extension_metrics_module.cc', # extensions hook for user metrics
   'safe_browsing_blocking_page.cc', # various interstitial types and actions
-  'language_options_handler.cc', # languages and input methods in chrome os
+  'language_options_handler_common.cc', # languages and input methods in CrOS
+  'cros_language_options_handler.cc', # languages and input methods in CrOS
   'about_flags.cc', # do not generate a warning; see AddAboutFlagsActions()
+  'external_metrics.cc',  # see AddChromeOSActions()
+  'core_options_handler.cc'  # see AddWebUIActions()
 )
 
 # Language codes used in Chrome. The list should be updated when a new
 # language is added to app/l10n_util.cc, as follows:
 #
 # % (cat app/l10n_util.cc | \
 #    perl -n0e 'print $1 if /kAcceptLanguageList.*?\{(.*?)\}/s' | \
 #    perl -nle 'print $1, if /"(.*)"/'; echo 'es-419') | \
 #   sort | perl -pe "s/(.*)\n/'\$1', /" | \
 #   fold -w75 -s | perl -pe 's/^/  /;s/ $//'; echo
@@ skipping 160 matching lines @@
 
   Arguments:
     path: path to the file
     actions: set of actions to add to
   """
   global number_of_files_total
   number_of_files_total = number_of_files_total + 1
   # we look for the UserMetricsAction structure constructor
   # this should be on one line
   action_re = re.compile(r'UserMetricsAction\("([^"]*)')
+  malformed_action_re = re.compile(r'[^a-zA-Z]UserMetricsAction\([^"]')
   computed_action_re = re.compile(r'UserMetrics::RecordComputedAction')
   line_number = 0
   for line in open(path):
     line_number = line_number + 1
     match = action_re.search(line)
     if match:  # Plain call to RecordAction
       actions.add(match.group(1))
+    elif malformed_action_re.search(line):
+      # Warn if this line is using RecordAction incorrectly.
+      print >>sys.stderr, ('WARNING: %s has malformed call to RecordAction'
+                           ' at %d' % (path, line_number))
     elif computed_action_re.search(line):
       # Warn if this file shouldn't be calling RecordComputedAction.
       if os.path.basename(path) not in KNOWN_COMPUTED_USERS:
-        print >>sys.stderr, 'WARNING: {0} has RecordComputedAction at {1}'.\
-                             format(path, line_number)
+        print >>sys.stderr, ('WARNING: %s has RecordComputedAction at %d' %
+                             (path, line_number))
 
-def WalkDirectory(root_path, actions):
+class WebUIActionsParser(HTMLParser):
+  """Parses an HTML file, looking for all tags with a 'metric' attribute.
+  Adds user actions corresponding to any metrics found.
+
+  Arguments:
+    actions: set of actions to add to
+  """
+  def __init__(self, actions):
+    HTMLParser.__init__(self)
+    self.actions = actions
+
+  def handle_starttag(self, tag, attrs):
+    # We only care to examine tags that have a 'metric' attribute.
+    attrs = dict(attrs)
+    if not 'metric' in attrs:
+      return
+
+    # Boolean metrics have two corresponding actions.  All other metrics have
+    # just one corresponding action.  By default, we check the 'dataType'
+    # attribute.
+    is_boolean = ('dataType' in attrs and attrs['dataType'] == 'boolean')
+    if 'type' in attrs and attrs['type'] in ('checkbox', 'radio'):
+      if attrs['type'] == 'checkbox':
+        # Checkboxes are boolean by default.  However, their 'value-type' can
+        # instead be set to 'integer'.
+        if 'value-type' not in attrs or attrs['value-type'] in ['', 'boolean']:
+          is_boolean = True
+      else:
+        # Radio buttons are boolean if and only if their values are 'true' or
+        # 'false'.
+        assert(attrs['type'] == 'radio')
+        if 'value' in attrs and attrs['value'] in ['true', 'false']:
+          is_boolean = True

[Ilya Sherman, 2011/07/07 21:01:01]

Actually, James, could you look at this bit of logic too?  I tried to distill
this from pref_ui.js, but I'm not sure I got it 100% correct.

[James Hawkins, 2011/07/07 21:10:08]

This part looks OK to me.

+
+    if is_boolean:
+      self.actions.add(attrs['metric'] + '_Enable')
+      self.actions.add(attrs['metric'] + '_Disable')
+    else:
+      self.actions.add(attrs['metric'])
+
+def GrepForWebUIActions(path, actions):
+  """Grep a WebUI source file for elements with associated metrics.
+
+  Arguments:
+    path: path to the file
+    actions: set of actions to add to
+  """
+  parser = WebUIActionsParser(actions)
+  parser.feed(open(path).read())
+  parser.close()
+
+def WalkDirectory(root_path, actions, extensions, callback):
   for path, dirs, files in os.walk(root_path):
     if '.svn' in dirs:
       dirs.remove('.svn')
     for file in files:
       ext = os.path.splitext(file)[1]
-      if ext in ('.cc', '.mm', '.c', '.m'):
-        GrepForActions(os.path.join(path, file), actions)
+      if ext in extensions:
+        callback(os.path.join(path, file), actions)
+
+def AddLiteralActions(actions):
+  """Add literal actions specified via calls to UserMetrics functions.
+
+  Arguments:
+    actions: set of actions to add to.
+  """
+  EXTENSIONS = ('.cc', '.mm', '.c', '.m')
+
+  # Walk the source tree to process all .cc files.
+  chrome_root = os.path.join(path_utils.ScriptDir(), '..')
+  WalkDirectory(chrome_root, actions, EXTENSIONS, GrepForActions)
+  content_root = os.path.join(path_utils.ScriptDir(), '..', '..', 'content')
+  WalkDirectory(content_root, actions, EXTENSIONS, GrepForActions)
+  webkit_root = os.path.join(path_utils.ScriptDir(), '..', '..', 'webkit')
+  WalkDirectory(os.path.join(webkit_root, 'glue'), actions, EXTENSIONS,
+                GrepForActions)
+  WalkDirectory(os.path.join(webkit_root, 'port'), actions, EXTENSIONS,
+                GrepForActions)
+
+def AddWebUIActions(actions):
+  """Add user actions defined in WebUI files.
+
+  Arguments:
+    actions: set of actions to add to.
+  """
+  resources_root = os.path.join(path_utils.ScriptDir(), '..', 'browser',
+                                'resources')
+  WalkDirectory(resources_root, actions, ('.html'), GrepForWebUIActions)
 
 def main(argv):
   if '--hash' in argv:
     hash_output = True
   else:
     hash_output = False
     print >>sys.stderr, "WARNING: If you added new UMA tags, you must" + \
            " use the --hash option to update chromeactions.txt."
   # if we do a hash output, we want to only append NEW actions, and we know
   # the file we want to work on
   actions = set()
 
   chromeactions_path = os.path.join(path_utils.ScriptDir(), "chromeactions.txt")
 
   if hash_output:
     f = open(chromeactions_path)
     for line in f:
       part = line.rpartition("\t")
       part = part[2].strip()
       actions.add(part)
     f.close()
 
 
   AddComputedActions(actions)
   # TODO(fmantek): bring back webkit editor actions.
   # AddWebKitEditorActions(actions)
   AddAboutFlagsActions(actions)
+  AddWebUIActions(actions)
 
-  # Walk the source tree to process all .cc files.
-  chrome_root = os.path.join(path_utils.ScriptDir(), '..')
-  WalkDirectory(chrome_root, actions)
-  content_root = os.path.join(path_utils.ScriptDir(), '..', '..', 'content')
-  WalkDirectory(content_root, actions)
-  webkit_root = os.path.join(path_utils.ScriptDir(), '..', '..', 'webkit')
-  WalkDirectory(os.path.join(webkit_root, 'glue'), actions)
-  WalkDirectory(os.path.join(webkit_root, 'port'), actions)
+  AddLiteralActions(actions)
 
   # print "Scanned {0} number of files".format(number_of_files_total)
   # print "Found {0} entries".format(len(actions))
 
   AddClosedSourceActions(actions)
   AddChromeOSActions(actions)
 
   if hash_output:
     f = open(chromeactions_path, "w")
 
 
   # Print out the actions as a sorted list.
   for action in sorted(actions):
     if hash_output:
       hash = hashlib.md5()
       hash.update(action)
       print >>f, '0x%s\t%s' % (hash.hexdigest()[:16], action)
     else:
       print action
 
   if hash_output:
     print "Done. Do not forget to add chromeactions.txt to your changelist"
 
 if '__main__' == __name__:
   main(sys.argv)