Add verbose support throught presubmit checks

presubmit_support.py

 #!/usr/bin/env python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Enables directory-specific presubmit checks to run at upload and/or commit.
 """
 
-__version__ = '1.5'
+__version__ = '1.6'
 
 # TODO(joi) Add caching where appropriate/needed. The API is designed to allow
 # caching (between all different invocations of presubmit scripts for a given
 # change). We should add it as our presubmit scripts start feeling slow.
 
 import cPickle  # Exposed through the API.
 import cStringIO  # Exposed through the API.
-import exceptions
 import fnmatch
 import glob
 import logging
 import marshal  # Exposed through the API.
 import optparse
 import os  # Somewhat exposed through the API.
 import pickle  # Exposed through the API.
 import random
 import re  # Exposed through the API.
 import sys  # Parts exposed through API.
@@ skipping 21 matching lines @@
 import owners
 import presubmit_canned_checks
 import scm
 import subprocess2 as subprocess  # Exposed through the API.
 
 
 # Ask for feedback only once in program lifetime.
 _ASKED_FOR_FEEDBACK = False
 
 
-class NotImplementedException(Exception):
-  """We're leaving placeholders in a bunch of places to remind us of the
-  design of the API, but we have not implemented all of it yet. Implement as
-  the need arises.
-  """
+class PresubmitFailure(Exception):
   pass
 
 
 def normpath(path):
   '''Version of os.path.normpath that also changes backward slashes to
   forward slashes when not running on Windows.
   '''
   # This is safe to always do because the Windows version of os.path.normpath
   # will replace forward slashes with backward slashes.
   path = path.replace(os.sep, '/')
@@ skipping 97 matching lines @@
     should_prompt = True
 
   class PresubmitNotifyResult(PresubmitResult):
     """Just print something to the screen -- but it's not even a warning."""
     pass
 
   class MailTextResult(PresubmitResult):
     """A warning that should be included in the review request email."""
     def __init__(self, *args, **kwargs):
       super(OutputApi.MailTextResult, self).__init__()
-      raise NotImplementedException()
+      raise NotImplementedError()
 
 
 class InputApi(object):
   """An instance of this object is passed to presubmit scripts so they can
   know stuff about the change they're looking at.
   """
   # Method could be a function
   # pylint: disable=R0201
 
   # File extensions that are considered source files from a style guide
@@ skipping 23 matching lines @@
       r".*\bsconsbuild[\\\/].*",
       # All caps files like README and LICENCE.
       r".*\b[A-Z0-9_]{2,}$",
       # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
       r"(|.*[\\\/])\.git[\\\/].*",
       r"(|.*[\\\/])\.svn[\\\/].*",
   )
 
   # TODO(dpranke): Update callers to pass in tbr, host_url, remove
   # default arguments.
-  def __init__(self, change, presubmit_path, is_committing, tbr, host_url=None):
+  def __init__(self, change, presubmit_path, is_committing, tbr, host_url,
+      verbose):
     """Builds an InputApi object.
 
     Args:
       change: A presubmit.Change object.
       presubmit_path: The path to the presubmit script being processed.
       is_committing: True if the change is about to be committed.
       tbr: True if '--tbr' was passed to skip any reviewer/owner checks
       host_url: scheme, host, and path of rietveld instance
     """
     # Version number of the presubmit_support script.
@@ skipping 33 matching lines @@
     # The local path of the currently-being-processed presubmit script.
     self._current_presubmit_path = os.path.dirname(presubmit_path)
 
     # We carry the canned checks so presubmit scripts can easily use them.
     self.canned_checks = presubmit_canned_checks
 
     # TODO(dpranke): figure out a list of all approved owners for a repo
     # in order to be able to handle wildcard OWNERS files?
     self.owners_db = owners.Database(change.RepositoryRoot(),
         fopen=file, os_path=self.os_path)
+    self.verbose = verbose
 
   def PresubmitLocalPath(self):
     """Returns the local path of the presubmit script currently being run.
 
     This is useful if you don't want to hard-code absolute paths in the
     presubmit script.  For example, It can be used to find another file
     relative to the PRESUBMIT.py script, so the whole tree can be branched and
     the presubmit script still works, without editing its content.
     """
     return self._current_presubmit_path
@@ skipping 299 matching lines @@
   # Method 'NNN' is abstract in class 'NNN' but is not overridden
   # pylint: disable=W0223
 
   def __init__(self, *args, **kwargs):
     AffectedFile.__init__(self, *args, **kwargs)
     self._server_path = None
     self._is_text_file = None
 
   def ServerPath(self):
     if self._server_path is None:
-      raise NotImplementedException()  # TODO(maruel) Implement.
+      raise NotImplementedError('TODO(maruel) Implement.')
     return self._server_path
 
   def IsDirectory(self):
     if self._is_directory is None:
       path = self.AbsoluteLocalPath()
       if os.path.exists(path):
         # Retrieve directly from the file system; it is much faster than
         # querying subversion, especially on Windows.
         self._is_directory = os.path.isdir(path)
       else:
-        # raise NotImplementedException()  # TODO(maruel) Implement.
         self._is_directory = False
     return self._is_directory
 
   def Property(self, property_name):
     if not property_name in self._properties:
-      raise NotImplementedException()  # TODO(maruel) Implement.
+      raise NotImplementedError('TODO(maruel) Implement.')
     return self._properties[property_name]
 
   def IsTextFile(self):
     if self._is_text_file is None:
       if self.Action() == 'D':
         # A deleted file is not a text file.
         self._is_text_file = False
       elif self.IsDirectory():
         self._is_text_file = False
       else:
-        # raise NotImplementedException()  # TODO(maruel) Implement.
         self._is_text_file = os.path.isfile(self.AbsoluteLocalPath())
     return self._is_text_file
 
   def GenerateScmDiff(self):
     return scm.GIT.GenerateDiff(self._local_root, files=[self.LocalPath(),])
 
 class Change(object):
   """Describe a change.
 
   Used directly by the presubmit scripts to query the current change being
@@ skipping 217 matching lines @@
     p = os.path.join(directory, 'PRESUBMIT.py')
     if os.path.isfile(p):
       results.append(p)
 
   logging.debug('Presubmit files: %s' % ','.join(results))
   return results
 
 
 class GetTrySlavesExecuter(object):
   @staticmethod
-  def ExecPresubmitScript(script_text):
+  def ExecPresubmitScript(script_text, presubmit_path):
     """Executes GetPreferredTrySlaves() from a single presubmit script.
 
     Args:
       script_text: The text of the presubmit script.
 
     Return:
       A list of try slaves.
     """
     context = {}
-    exec script_text in context
+    try:
+      exec script_text in context
+    except Exception, e:
+      raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
 
     function_name = 'GetPreferredTrySlaves'
     if function_name in context:
       result = eval(function_name + '()', context)
       if not isinstance(result, types.ListType):
-        raise exceptions.RuntimeError(
+        raise PresubmitFailure(
             'Presubmit functions must return a list, got a %s instead: %s' %
             (type(result), str(result)))
       for item in result:
         if not isinstance(item, basestring):
-          raise exceptions.RuntimeError('All try slaves names must be strings.')
+          raise PresubmitFailure('All try slaves names must be strings.')
         if item != item.strip():
-          raise exceptions.RuntimeError('Try slave names cannot start/end'
-                                        'with whitespace')
+          raise PresubmitFailure(
+              'Try slave names cannot start/end with whitespace')
     else:
       result = []
     return result
 
 
 def DoGetTrySlaves(changed_files,
                    repository_root,
                    default_presubmit,
                    verbose,
                    output_stream):
@@ skipping 10 matching lines @@
     List of try slaves
   """
   presubmit_files = ListRelevantPresubmitFiles(changed_files, repository_root)
   if not presubmit_files and verbose:
     output_stream.write("Warning, no presubmit.py found.\n")
   results = []
   executer = GetTrySlavesExecuter()
   if default_presubmit:
     if verbose:
       output_stream.write("Running default presubmit script.\n")
-    results += executer.ExecPresubmitScript(default_presubmit)
+    fake_path = os.path.join(repository_root, 'PRESUBMIT.py')
+    results += executer.ExecPresubmitScript(default_presubmit, fake_path)
   for filename in presubmit_files:
     filename = os.path.abspath(filename)
     if verbose:
       output_stream.write("Running %s\n" % filename)
     # Accept CRLF presubmit script.
     presubmit_script = gclient_utils.FileRead(filename, 'rU')
-    results += executer.ExecPresubmitScript(presubmit_script)
+    results += executer.ExecPresubmitScript(presubmit_script, filename)
 
   slaves = list(set(results))
   if slaves and verbose:
     output_stream.write(', '.join(slaves))
     output_stream.write('\n')
   return slaves
 
 
 class PresubmitExecuter(object):
-  def __init__(self, change, committing, tbr, host_url):
+  def __init__(self, change, committing, tbr, host_url, verbose):
     """
     Args:
       change: The Change object.
       committing: True if 'gcl commit' is running, False if 'gcl upload' is.
       tbr: True if '--tbr' was passed to skip any reviewer/owner checks
       host_url: scheme, host, and path of rietveld instance
           (or None for default)
     """
     self.change = change
     self.committing = committing
     self.tbr = tbr
     self.host_url = host_url
+    self.verbose = verbose
 
   def ExecPresubmitScript(self, script_text, presubmit_path):
     """Executes a single presubmit script.
 
     Args:
       script_text: The text of the presubmit script.
       presubmit_path: The path to the presubmit file (this will be reported via
         input_api.PresubmitLocalPath()).
 
     Return:
       A list of result objects, empty if no problems.
     """
 
     # Change to the presubmit file's directory to support local imports.
     main_path = os.getcwd()
     os.chdir(os.path.dirname(presubmit_path))
 
     # Load the presubmit script into context.
     input_api = InputApi(self.change, presubmit_path, self.committing,
-                         self.tbr, self.host_url)
+                         self.tbr, self.host_url, self.verbose)
     context = {}
-    exec script_text in context
+    try:
+      exec script_text in context
+    except Exception, e:
+      raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
 
     # These function names must change if we make substantial changes to
     # the presubmit API that are not backwards compatible.
     if self.committing:
       function_name = 'CheckChangeOnCommit'
     else:
       function_name = 'CheckChangeOnUpload'
     if function_name in context:
       context['__args'] = (input_api, OutputApi())
       logging.debug('Running %s in %s' % (function_name, presubmit_path))
       result = eval(function_name + '(*__args)', context)
       logging.debug('Running %s done.' % function_name)
       if not (isinstance(result, types.TupleType) or
               isinstance(result, types.ListType)):
-        raise exceptions.RuntimeError(
+        raise PresubmitFailure(
           'Presubmit functions must return a tuple or list')
       for item in result:
         if not isinstance(item, OutputApi.PresubmitResult):
-          raise exceptions.RuntimeError(
+          raise PresubmitFailure(
             'All presubmit results must be of types derived from '
             'output_api.PresubmitResult')
     else:
       result = ()  # no error since the script doesn't care about current event.
 
     # Return the process to the original working directory.
     os.chdir(main_path)
     return result
 
 
@@ skipping 40 matching lines @@
   if committing:
     output.write("Running presubmit commit checks ...\n")
   else:
     output.write("Running presubmit upload checks ...\n")
   start_time = time.time()
   presubmit_files = ListRelevantPresubmitFiles(change.AbsoluteLocalPaths(True),
                                                change.RepositoryRoot())
   if not presubmit_files and verbose:
     output.write("Warning, no presubmit.py found.\n")
   results = []
-  executer = PresubmitExecuter(change, committing, tbr, host_url)
+  executer = PresubmitExecuter(change, committing, tbr, host_url, verbose)
   if default_presubmit:
     if verbose:
       output.write("Running default presubmit script.\n")
     fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
     results += executer.ExecPresubmitScript(default_presubmit, fake_path)
   for filename in presubmit_files:
     filename = os.path.abspath(filename)
     if verbose:
       output.write("Running %s\n" % filename)
     # Accept CRLF presubmit script.
@@ skipping 92 matching lines @@
 
 def Main(argv):
   parser = optparse.OptionParser(usage="%prog [options] <files...>",
                                  version="%prog " + str(__version__))
   parser.add_option("-c", "--commit", action="store_true", default=False,
                    help="Use commit instead of upload checks")
   parser.add_option("-u", "--upload", action="store_false", dest='commit',
                    help="Use upload instead of commit checks")
   parser.add_option("-r", "--recursive", action="store_true",
                    help="Act recursively")
-  parser.add_option("-v", "--verbose", action="store_true", default=False,
-                   help="Verbose output")
+  parser.add_option("-v", "--verbose", action="count", default=0,
+                   help="Use 2 times for more debug info")
   parser.add_option("--name", default='no name')
   parser.add_option("--description", default='')
   parser.add_option("--issue", type='int', default=0)
   parser.add_option("--patchset", type='int', default=0)
   parser.add_option("--root", default=os.getcwd(),
                     help="Search for PRESUBMIT.py up to this directory. "
                     "If inherit-review-settings-ok is present in this "
                     "directory, parent directories up to the root file "
                     "system directories will also be searched.")
   parser.add_option("--default_presubmit")
   parser.add_option("--may_prompt", action='store_true', default=False)
   options, args = parser.parse_args(argv)
-  if options.verbose:
+  if options.verbose >= 2:
     logging.basicConfig(level=logging.DEBUG)
+  elif options.verbose:
+    logging.basicConfig(level=logging.INFO)
+  else:
+    logging.basicConfig(level=logging.ERROR)
   change_class, files = load_files(options, args)
   if not change_class:
     parser.error('For unversioned directory, <files> is not optional.')
-  if options.verbose:
-    print "Found %d file(s)." % len(files)
-  results = DoPresubmitChecks(change_class(options.name,
-                                           options.description,
-                                           options.root,
-                                           files,
-                                           options.issue,
-                                           options.patchset),
-                              options.commit,
-                              options.verbose,
-                              sys.stdout,
-                              sys.stdin,
-                              options.default_presubmit,
-                              options.may_prompt)
-  return not results.should_continue()
+  logging.info('Found %d file(s).' % len(files))
+  try:
+    results = DoPresubmitChecks(
+        change_class(options.name,
+                    options.description,
+                    options.root,
+                    files,
+                    options.issue,
+                    options.patchset),
+        options.commit,
+        options.verbose,
+        sys.stdout,
+        sys.stdin,
+        options.default_presubmit,
+        options.may_prompt)
+    return not results.should_continue()
+  except PresubmitFailure, e:
+    print >> sys.stderr, e
+    print >> sys.stderr, 'Maybe your depot_tools is out of date?'
+    print >> sys.stderr, 'If all fails, contact maruel@'
+    return 2
 
 
 if __name__ == '__main__':
   fix_encoding.fix_encoding()
   sys.exit(Main(None))