Merge remote branch 'cros/upstream' into master

utils/check_patch.py

 #!/usr/bin/python
 """
 Script to verify errors on autotest code contributions (patches).
 The workflow is as follows:
 
  * Patch will be applied and eventual problems will be notified.
  * If there are new files created, remember user to add them to VCS.
  * If any added file looks like a executable file, remember user to make them
    executable.
  * If any of the files added or modified introduces trailing whitespaces, tabs
    or incorrect indentation, report problems.
  * If any of the files have problems during pylint validation, report failures.
  * If any of the files changed have a unittest suite, run the unittest suite
    and report any failures.
 
 Usage: check_patch.py -p [/path/to/patch]
        check_patch.py -i [patchwork id]
 
 @copyright: Red Hat Inc, 2009.
 @author: Lucas Meneghel Rodrigues <lmr@redhat.com>
 """
 
-import os, stat, logging, sys, optparse
+import os, stat, logging, sys, optparse, time
 import common
 from autotest_lib.client.common_lib import utils, error, logging_config
 from autotest_lib.client.common_lib import logging_manager
 
 
 class CheckPatchLoggingConfig(logging_config.LoggingConfig):
     def configure_logging(self, results_dir=None, verbose=False):
         super(CheckPatchLoggingConfig, self).configure_logging(use_console=True,
                                                                verbose=verbose)
 
 
+def ask(question, auto=False):
+    """
+    Raw input with a prompt that emulates logging.
+
+    @param question: Question to be asked
+    @param auto: Whether to return "y" instead of asking the question
+    """
+    if auto:
+        logging.info("%s (y/n) y" % question)
+        return "y"
+    return raw_input("%s INFO | %s (y/n) " %
+                     (time.strftime("%H:%M:%S", time.localtime()), question))
+
+
 class VCS(object):
     """
     Abstraction layer to the version control system.
     """
     def __init__(self):
         """
         Class constructor. Guesses the version control name and instantiates it
         as a backend.
         """
         backend_name = self.guess_vcs_name()
@@ skipping 52 matching lines @@
         return self.backend.update()
 
 
 class SubVersionBackend(object):
     """
     Implementation of a subversion backend for use with the VCS abstraction
     layer.
     """
     def __init__(self):
         logging.debug("Subversion VCS backend initialized.")
+        self.ignored_extension_list = ['.orig', '.bak']
 
 
     def get_unknown_files(self):
         status = utils.system_output("svn status --ignore-externals")
         unknown_files = []
         for line in status.split("\n"):
             status_flag = line[0]
             if line and status_flag == "?":
-                unknown_files.append(line[1:].strip())
+                for extension in self.ignored_extension_list:
+                    if not line.endswith(extension):
+                        unknown_files.append(line[1:].strip())
         return unknown_files
 
 
     def get_modified_files(self):
         status = utils.system_output("svn status --ignore-externals")
         modified_files = []
         for line in status.split("\n"):
             status_flag = line[0]
             if line and status_flag == "M" or status_flag == "A":
                 modified_files.append(line[1:].strip())
@@ skipping 48 matching lines @@
             utils.system("svn update", ignore_status=True)
         except error.CmdError, e:
             logging.error("SVN tree update failed: %s" % e)
 
 
 class FileChecker(object):
     """
     Picks up a given file and performs various checks, looking after problems
     and eventually suggesting solutions.
     """
-    def __init__(self, path):
+    def __init__(self, path, confirm=False):
         """
         Class constructor, sets the path attribute.
 
         @param path: Path to the file that will be checked.
+        @param confirm: Whether to answer yes to all questions asked without
+                prompting the user.
         """
         self.path = path
+        self.confirm = confirm
         self.basename = os.path.basename(self.path)
         if self.basename.endswith('.py'):
             self.is_python = True
         else:
             self.is_python = False
 
         mode = os.stat(self.path)[stat.ST_MODE]
         if mode & stat.S_IXUSR:
             self.is_executable = True
         else:
             self.is_executable = False
 
         checked_file = open(self.path, "r")
         self.first_line = checked_file.readline()
         checked_file.close()
         self.corrective_actions = []
-        self.indentation_exceptions = ['cli/job_unittest.py']
+        self.indentation_exceptions = ['job_unittest.py']
 
 
     def _check_indent(self):
         """
         Verifies the file with reindent.py. This tool performs the following
         checks on python files:
 
           * Trailing whitespaces
           * Tabs
           * End of line
           * Incorrect indentation
 
         For the purposes of checking, the dry run mode is used and no changes
         are made. It is up to the user to decide if he wants to run reindent
         to correct the issues.
         """
         reindent_raw = utils.system_output('reindent.py -v -d %s | head -1' %
                                            self.path)
         reindent_results = reindent_raw.split(" ")[-1].strip(".")
         if reindent_results == "changed":
             if self.basename not in self.indentation_exceptions:
-                logging.error("Possible indentation and spacing issues on "
-                              "file %s" % self.path)
                 self.corrective_actions.append("reindent.py -v %s" % self.path)
 
 
     def _check_code(self):
         """
         Verifies the file with run_pylint.py. This tool will call the static
         code checker pylint using the special autotest conventions and warn
         only on problems. If problems are found, a report will be generated.
         Some of the problems reported might be bogus, but it's allways good
         to look at them.
         """
         c_cmd = 'run_pylint.py %s' % self.path
         rc = utils.system(c_cmd, ignore_status=True)
         if rc != 0:
-            logging.error("Possible syntax problems on file %s", self.path)
-            logging.error("You might want to rerun '%s'", c_cmd)
+            logging.error("Syntax issues found during '%s'", c_cmd)
 
 
     def _check_unittest(self):
         """
         Verifies if the file in question has a unittest suite, if so, run the
         unittest and report on any failures. This is important to keep our
         unit tests up to date.
         """
         if "unittest" not in self.basename:
             stripped_name = self.basename.strip(".py")
             unittest_name = stripped_name + "_unittest.py"
             unittest_path = self.path.replace(self.basename, unittest_name)
             if os.path.isfile(unittest_path):
                 unittest_cmd = 'python %s' % unittest_path
                 rc = utils.system(unittest_cmd, ignore_status=True)
                 if rc != 0:
-                    logging.error("Problems during unit test execution "
-                                  "for file %s", self.path)
-                    logging.error("You might want to rerun '%s'", unittest_cmd)
+                    logging.error("Unittest issues found during '%s'",
+                                  unittest_cmd)
 
 
     def _check_permissions(self):
         """
         Verifies the execution permissions, specifically:
           * Files with no shebang and execution permissions are reported.
           * Files with shebang and no execution permissions are reported.
         """
         if self.first_line.startswith("#!"):
             if not self.is_executable:
-                logging.info("File %s seems to require execution "
-                             "permissions. ", self.path)
-                self.corrective_actions.append("chmod +x %s" % self.path)
+                self.corrective_actions.append("svn propset svn:executable ON %s" % self.path)
         else:
             if self.is_executable:
-                logging.info("File %s does not seem to require execution "
-                             "permissions. ", self.path)
-                self.corrective_actions.append("chmod -x %s" % self.path)
+                self.corrective_actions.append("svn propdel svn:executable %s" % self.path)
 
 
     def report(self):
         """
         Executes all required checks, if problems are found, the possible
         corrective actions are listed.
         """
         self._check_permissions()
         if self.is_python:
             self._check_indent()
             self._check_code()
             self._check_unittest()
         if self.corrective_actions:
-            logging.info("The following corrective actions are suggested:")
             for action in self.corrective_actions:
-                logging.info(action)
-                answer = raw_input("Would you like to apply it? (y/n) ")
+                answer = ask("Would you like to execute %s?" % action,
+                             auto=self.confirm)
                 if answer == "y":
                     rc = utils.system(action, ignore_status=True)
                     if rc != 0:
                         logging.error("Error executing %s" % action)
 
 
 class PatchChecker(object):
-    def __init__(self, patch=None, patchwork_id=None):
+    def __init__(self, patch=None, patchwork_id=None, confirm=False):
+        self.confirm = confirm
         self.base_dir = os.getcwd()
         if patch:
             self.patch = os.path.abspath(patch)
         if patchwork_id:
             self.patch = self._fetch_from_patchwork(patchwork_id)
 
         if not os.path.isfile(self.patch):
             logging.error("Invalid patch file %s provided. Aborting.",
                           self.patch)
             sys.exit(1)
 
         self.vcs = VCS()
         changed_files_before = self.vcs.get_modified_files()
         if changed_files_before:
             logging.error("Repository has changed files prior to patch "
                           "application. ")
-            answer = raw_input("Would you like to revert them? (y/n) ")
+            answer = ask("Would you like to revert them?", auto=self.confirm)
             if answer == "n":
                 logging.error("Not safe to proceed without reverting files.")
                 sys.exit(1)
             else:
                 for changed_file in changed_files_before:
                     self.vcs.revert_file(changed_file)
 
         self.untracked_files_before = self.vcs.get_unknown_files()
         self.vcs.update()
 
@@ skipping 27 matching lines @@
         add_to_vcs = []
         for untracked_file in untracked_files_after:
             if untracked_file not in self.untracked_files_before:
                 add_to_vcs.append(untracked_file)
 
         if add_to_vcs:
             logging.info("The files: ")
             for untracked_file in add_to_vcs:
                 logging.info(untracked_file)
             logging.info("Might need to be added to VCS")
-            logging.info("Would you like to add them to VCS ? (y/n/abort) ")
-            answer = raw_input()
+            answer = ask("Would you like to add them to VCS ?")
             if answer == "y":
                 for untracked_file in add_to_vcs:
                     self.vcs.add_untracked_file(untracked_file)
                     modified_files_after.append(untracked_file)
             elif answer == "n":
                 pass
-            elif answer == "abort":
-                sys.exit(1)
 
         for modified_file in modified_files_after:
-            file_checker = FileChecker(modified_file)
-            file_checker.report()
+            # Additional safety check, new commits might introduce
+            # new directories
+            if os.path.isfile(modified_file):
+                file_checker = FileChecker(modified_file)
+                file_checker.report()
 
 
     def check(self):
         self.vcs.apply_patch(self.patch)
         self._check_files_modified_patch()
 
 
 if __name__ == "__main__":
     parser = optparse.OptionParser()
     parser.add_option('-p', '--patch', dest="local_patch", action='store',
                       help='path to a patch file that will be checked')
     parser.add_option('-i', '--patchwork-id', dest="id", action='store',
                       help='id of a given patchwork patch')
     parser.add_option('--verbose', dest="debug", action='store_true',
                       help='include debug messages in console output')
+    parser.add_option('-f', '--full-check', dest="full_check",
+                      action='store_true',
+                      help='check the full tree for corrective actions')
+    parser.add_option('-y', '--yes', dest="confirm",
+                      action='store_true',
+                      help='Answer yes to all questions')
 
     options, args = parser.parse_args()
     local_patch = options.local_patch
     id = options.id
     debug = options.debug
+    full_check = options.full_check
+    confirm = options.confirm
 
     logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug)
 
-    if local_patch:
-        patch_checker = PatchChecker(patch=local_patch)
-    elif id:
-        patch_checker = PatchChecker(patchwork_id=id)
+    ignore_file_list = ['common.py']
+    if full_check:
+        for root, dirs, files in os.walk('.'):
+            if not '.svn' in root:
+                for file in files:
+                    if file not in ignore_file_list:
+                        path = os.path.join(root, file)
+                        file_checker = FileChecker(path, confirm=confirm)
+                        file_checker.report()
     else:
-        logging.error('No patch or patchwork id specified. Aborting.')
-        sys.exit(1)
-
-    patch_checker.check()
+        if local_patch:
+            patch_checker = PatchChecker(patch=local_patch, confirm=confirm)
+        elif id:
+            patch_checker = PatchChecker(patchwork_id=id, confirm=confirm)
+        else:
+            logging.error('No patch or patchwork id specified. Aborting.')
+            sys.exit(1)
+        patch_checker.check()