Modify rebaseline tool to work with git and add vista rebaseline...

webkit/tools/layout_tests/rebaseline.py

 #!/usr/bin/env python
 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Rebaselining tool that automatically produces baselines for all platforms.
 
 The script does the following for each platform specified:
   1. Compile a list of tests that need rebaselining.
   2. Download test result archive from buildbot for the platform.
@@ skipping 17 matching lines @@
 import time
 import urllib
 import webbrowser
 import zipfile
 
 from layout_package import path_utils
 from layout_package import test_expectations
 from test_types import image_diff
 from test_types import text_diff
 
+# Repository type constants.
+REPO_SVN, REPO_UNKNOWN = range(2)
+
 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
-REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'linux']
+REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
 ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel',
                          'win-vista': 'webkit-dbg-vista',
                          'win-xp': 'webkit-rel',
                          'mac': 'webkit-rel-mac5',
                          'linux': 'webkit-rel-linux',
                          'win-canary': 'webkit-rel-webkit-org',
                          'win-vista-canary': 'webkit-dbg-vista',
                          'win-xp-canary': 'webkit-rel-webkit-org',
                          'mac-canary': 'webkit-rel-mac-webkit-org',
                          'linux-canary': 'webkit-rel-linux-webkit-org'}
 
-def RunShell(command, print_output=False):
-  """Executes a command and returns the output.
+def RunShellWithReturnCode(command, print_output=False):
+  """Executes a command and returns the output and process return code.
 
   Args:
     command: program and arguments.
     print_output: if true, print the command results to standard output.
 
   Returns:
-    command output
+    command output, return code
   """
 
   # Use a shell for subcommands on Windows to get a PATH search.
   use_shell = sys.platform.startswith('win')
   p = subprocess.Popen(command, stdout=subprocess.PIPE,
                        stderr=subprocess.STDOUT, shell=use_shell)
   if print_output:
     output_array = []
     while True:
       line = p.stdout.readline()
       if not line:
         break
       if print_output:
         print line.strip('\n')
       output_array.append(line)
     output = ''.join(output_array)
   else:
     output = p.stdout.read()
   p.wait()
   p.stdout.close()
+
+  return output, p.returncode
+
+def RunShell(command, print_output=False):
+  """Executes a command and returns the output.
+
+  Args:
+    command: program and arguments.
+    print_output: if true, print the command results to standard output.
+
+  Returns:
+    command output
+  """
+
+  output, return_code = RunShellWithReturnCode(command, print_output)
   return output
 
-
 def LogDashedString(text, platform, logging_level=logging.INFO):
   """Log text message with dashes on both sides."""
 
   msg = text
   if platform:
     msg += ': ' + platform
   if len(msg) < 78:
     dashes = '-' * ((78 - len(msg)) / 2)
     msg = '%s %s %s' % (dashes, msg, dashes)
 
@@ skipping 72 matching lines @@
 
     # Create tests and expectations helper which is used to:
     #   -. compile list of tests that need rebaselining.
     #   -. update the tests in test_expectations file after rebaseline is done.
     self._test_expectations = test_expectations.TestExpectations(None,
                                                                  self._file_dir,
                                                                  platform,
                                                                  False,
                                                                  False)
 
+    self._repo_type = self._GetRepoType()
+
   def Run(self, backup):
     """Run rebaseline process."""
 
     LogDashedString('Compiling rebaselining tests', self._platform)
     if not self._CompileRebaseliningTests():
       return True
 
     LogDashedString('Downloading archive', self._platform)
     archive_file = self._DownloadBuildBotArchive()
     logging.info('')
@@ skipping 19 matching lines @@
       return False
 
     logging.warning('All tests needing rebaselining were successfully '
                     'rebaselined.')
 
     return True
 
   def GetRebaseliningTests(self):
     return self._rebaselining_tests
 
+  def _GetRepoType(self):
+    """Get the repository type that client is using."""
+
+    output, return_code = RunShellWithReturnCode(['svn', 'info'], False)
+    if return_code == 0:
+      return REPO_SVN
+
+    return REPO_UNKNOWN
+
   def _CompileRebaseliningTests(self):
     """Compile list of tests that need rebaselining for the platform.
 
     Returns:
       List of tests that need rebaselining or
       None if there is no such test.
     """
 
     self._rebaselining_tests = self._test_expectations.GetRebaseliningFailures()
     if not self._rebaselining_tests:
@@ skipping 130 matching lines @@
                                                 test_basename,
                                                 suffix)
         logging.debug('  Archive test file name: "%s"', archive_test_name)
         if not archive_test_name in zip_namelist:
           logging.info('  %s file not in archive.', suffix)
           continue
 
         found = True
         logging.info('  %s file found in archive.', suffix)
 
+        # Extract new baseline from archive and save it to a temp file.
+        data = zip_file.read(archive_test_name)
+        temp_fd, temp_name = tempfile.mkstemp(suffix)
+        f = os.fdopen(temp_fd, 'wb')
+        f.write(data)
+        f.close()
+
         expected_filename = '%s-expected%s' % (test_basename, suffix)
         expected_fullpath = os.path.join(
             path_utils.ChromiumBaselinePath(platform), expected_filename)
         expected_fullpath = os.path.normpath(expected_fullpath)
         logging.debug('  Expected file full path: "%s"', expected_fullpath)
 
-        data = zip_file.read(archive_test_name)
+        # TODO(victorw): for now, the rebaselining tool checks whether
+        # or not THIS baseline is duplicate and should be skipped.
+        # We could improve the tool to check all baselines in upper and lower
+        # levels and remove all duplicated baselines.
+        if self._IsDupBaseline(temp_name,
+                               expected_fullpath,
+                               test,
+                               suffix,
+                               self._platform):
+          os.remove(temp_name)
+          self._DeleteBaseline(expected_fullpath)
+          continue
 
         # Create the new baseline directory if it doesn't already exist.
         path_utils.MaybeMakeDirectory(os.path.dirname(expected_fullpath))
 
-        f = open(expected_fullpath, 'wb')
-        f.write(data)
-        f.close()
-
-        # TODO(victorw): for now, the rebaselining tool checks whether
-        # or not THIS baseline is duplicate and should be skipped.
-        # We could improve the tool to check all baselines in upper and lower
-        # levels and remove all duplicated baselines.
-        if self._IsDupBaseline(expected_fullpath, test, suffix, self._platform):
-          # Clean up the duplicate baseline.
-          self._DeleteBaseline(expected_fullpath)
-          continue
+        shutil.move(temp_name, expected_fullpath)
 
         if not self._SvnAdd(expected_fullpath):
           svn_error = True
         elif suffix != '.checksum':
           self._CreateHtmlBaselineFiles(expected_fullpath)
 
       if not found:
         logging.warn('  No new baselines found in archive.')
       else:
         if svn_error:
           logging.warn('  Failed to add baselines to SVN.')
         else:
           logging.info('  Rebaseline succeeded.')
           self._rebaselined_tests.append(test)
 
       test_no += 1
 
     zip_file.close()
     os.remove(archive_file)
 
     return self._rebaselined_tests
 
-  def _IsDupBaseline(self, baseline_path, test, suffix, platform):
+  def _IsDupBaseline(self, new_baseline, baseline_path, test, suffix, platform):
     """Check whether a baseline is duplicate and can fallback to same
        baseline for another platform. For example, if a test has same baseline
        on linux and windows, then we only store windows baseline and linux
        baseline will fallback to the windows version.
 
     Args:
       expected_filename: baseline expectation file name.
       test: test name.
       suffix: file suffix of the expected results, including dot; e.g. '.txt'
               or '.png'.
       platform: baseline platform 'mac', 'win' or 'linux'.
 
     Returns:
       True if the baseline is unnecessary.
       False otherwise.
     """
     test_filepath = os.path.join(path_utils.LayoutTestsDir(), test)
     all_baselines = path_utils.ExpectedBaseline(test_filepath,
                                                 suffix,
                                                 platform,
                                                 True)
     for (fallback_dir, fallback_file) in all_baselines:
       if fallback_dir and fallback_file:
         fallback_fullpath = os.path.normpath(
             os.path.join(fallback_dir, fallback_file))
         if fallback_fullpath.lower() != baseline_path.lower():
-          if not self._DiffBaselines(baseline_path, fallback_fullpath):
+          if not self._DiffBaselines(new_baseline, fallback_fullpath):
             logging.info('  Found same baseline at %s', fallback_fullpath)
             return True
           else:
             return False
 
     return False
 
   def _DiffBaselines(self, file1, file2):
     """Check whether two baselines are different.
 
@@ skipping 13 matching lines @@
       return True
 
     if ext1 == '.PNG':
       return image_diff.ImageDiff(self._platform, '').DiffFiles(file1,
                                                                 file2)
     else:
       return text_diff.TestTextDiff(self._platform, '').DiffFiles(file1,
                                                                   file2)
 
   def _DeleteBaseline(self, filename):
-    """Remove the file from SVN repository and delete it from disk.
+    """Remove the file from repository and delete it from disk.
 
     Args:
       filename: full path of the file to delete.
     """
 
-    if not filename:
+    if not filename or not os.path.isfile(filename):
       return
 
-    parent_dir, basename = os.path.split(filename)
-    original_dir = os.getcwd()
-    os.chdir(parent_dir)
-    status_output = RunShell(['svn', 'delete', '--force', basename], False)
-    os.chdir(original_dir)
+    if self._repo_type == REPO_SVN:
+      parent_dir, basename = os.path.split(filename)
+      original_dir = os.getcwd()
+      os.chdir(parent_dir)
+      RunShell(['svn', 'delete', '--force', basename], False)
+      os.chdir(original_dir)
+    else:
+      os.remove(filename)
 
   def _UpdateRebaselinedTestsInFile(self, backup):
     """Update the rebaselined tests in test expectations file.
 
     Args:
       backup: if True, backup the original test expectations file.
 
     Returns:
       no
     """
@@ skipping 13 matching lines @@
 
     Returns:
       True if the file already exists in SVN or is sucessfully added to SVN.
       False otherwise.
     """
 
     if not filename:
       return False
 
     parent_dir, basename = os.path.split(filename)
-    if parent_dir == filename:
-      logging.info("No svn checkout found. Assuming it's a git repo and not "
-                   "adding")
+    if self._repo_type != REPO_SVN or parent_dir == filename:
+      logging.info("No svn checkout found, skip svn add.")
       return True
 
     original_dir = os.getcwd()
     os.chdir(parent_dir)
     status_output = RunShell(['svn', 'status', basename], False)
     os.chdir(original_dir)
     output = status_output.upper()
     if output.startswith('A') or output.startswith('M'):
       logging.info('  File already added to SVN: "%s"', filename)
       return True
@@ skipping 328 matching lines @@
 def main():
   """Main function to produce new baselines."""
 
   option_parser = optparse.OptionParser()
   option_parser.add_option('-v', '--verbose',
                            action='store_true',
                            default=False,
                            help='include debug-level logging.')
 
   option_parser.add_option('-p', '--platforms',
-                           default='mac,win,win-xp,linux',
+                           default='mac,win,win-xp,win-vista,linux',
                            help=('Comma delimited list of platforms that need '
                                  'rebaselining.'))
 
   option_parser.add_option('-u', '--archive_url',
                            default=('http://build.chromium.org/buildbot/'
                                     'layout_test_results'),
                            help=('Url to find the layout test result archive '
                                  'file.'))
 
   option_parser.add_option('-t', '--archive_name',
@@ skipping 93 matching lines @@
                                    rebaseline_platforms,
                                    rebaselining_tests)
     html_generator.GenerateHtml()
     html_generator.ShowHtml()
     LogDashedString('Rebaselining result comparison done', None)
 
   sys.exit(0)
 
 if '__main__' == __name__:
   main()