svndiff.py: add support for git checkouts

tools/svndiff.py

 #!/usr/bin/python

[epoger, 2013/07/17 04:38:25]

Tested with JSON expectations as follows:

# Revert any pending changes in my json-expectations directory
EXPECTATIONS_DIR=~/src/skia-git-trunk/trunk/experimental/json-expectations
pushd $EXPECTATIONS_DIR ; git checkout -- */*.json ; popd
pushd $EXPECTATIONS_DIR ; git status ; popd

# Update my json-expectations directory
tools/rebaseline.py --expectations-root $EXPECTATIONS_DIR
pushd $EXPECTATIONS_DIR ; git status ; popd

# Run the modified svndiff script, and view the HTML diffs
TMPDIR=/tmp/jsondiffs
rm -rf $TMPDIR && \
  mkdir -p $TMPDIR && \
  tools/svndiff.py --dest-dir $TMPDIR --source-dir $EXPECTATIONS_DIR

 '''
 Copyright 2012 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 '''
 
 '''
-Generates a visual diff of all pending changes in the local SVN checkout.
+Generates a visual diff of all pending changes in the local SVN (or git!)
+checkout.
 
 Launch with --help to see more information.
 
+TODO(epoger): Now that this tool supports either git or svn, rename it.
 TODO(epoger): Fix indentation in this file (2-space indents, not 4-space).
 '''
 
 # common Python modules
 import optparse
 import os
 import re
 import shutil
+import subprocess
 import sys
 import tempfile
 import urllib2
 
 # Imports from within Skia
 #
 # We need to add the 'gm' directory, so that we can import gm_json.py within
 # that directory.  That script allows us to parse the actual-results.json file
 # written out by the GM tool.
 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
 # so any dirs that are already in the PYTHONPATH will be preferred.
 #
 # This assumes that the 'gm' directory has been checked out as a sibling of
 # the 'tools' directory containing this script, which will be the case if
 # 'trunk' was checked out as a single unit.
 GM_DIRECTORY = os.path.realpath(
     os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
 if GM_DIRECTORY not in sys.path:
     sys.path.append(GM_DIRECTORY)
 import gm_json
 import jsondiff
 import svn
 
 USAGE_STRING = 'Usage: %s [options]'
 HELP_STRING = '''
 
-Generates a visual diff of all pending changes in the local SVN checkout.
+Generates a visual diff of all pending changes in the local SVN/git checkout.
 
 This includes a list of all files that have been added, deleted, or modified
-(as far as SVN knows about).  For any image modifications, pixel diffs will
+(as far as SVN/git knows about).  For any image modifications, pixel diffs will
 be generated.
 
 '''
 
 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
 
 TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir)
 
 OPTION_DEST_DIR = '--dest-dir'
 OPTION_PATH_TO_SKDIFF = '--path-to-skdiff'
@@ skipping 91 matching lines @@
         if new_checksum:
             new_image_url = _CreateGSUrl(
                 imagename=imagename,
                 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
                 hash_digest=new_checksum)
             _DownloadUrlToFile(
                 source_url=new_image_url,
                 dest_path=os.path.join(new_flattened_dir,
                                        filename_prefix + imagename))
 
+def _RunCommand(args):
+    """Run a command (from self._directory) and return stdout as a single
+    string.
+
+    @param args a list of arguments
+    """
+    proc = subprocess.Popen(args,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    (stdout, stderr) = proc.communicate()
+    if proc.returncode is not 0:
+        raise Exception('command "%s" failed: %s' % (args, stderr))
+    return stdout
+
+def _GitGetModifiedFiles():
+    """Returns a list of locally modified files within the current working dir.
+
+    TODO(epoger): Move this into a git utility package?
+    """
+    status_regex_string = '^...(\S+)'
+    stdout = _RunCommand(['git', 'status', '-s'])

[Stephen White, 2013/07/17 10:44:40]

GitNit(tm): git status is a "porcelain" command, and isn't intended to be called
from scripts. I think the equivalent plumbing command for your purposes would be
something like "git ls-files -m".

[epoger, 2013/07/18 15:00:04]

Ah, thanks for teaching this git n00b something.  I found this handy overview:
http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain
Done, and confirmed that it works.

+    status_regex = re.compile(status_regex_string, re.MULTILINE)
+    return status_regex.findall(stdout)
+
+def _GitExportBaseVersionOfFile(file_within_repo, dest_path):
+    """Retrieves a copy of the base version of a file within the repository.
+
+    @param file_within_repo path to the file within the repo whose base
+           version you wish to obtain
+    @param dest_path destination to which to write the base content
+
+    TODO(epoger): Move this into a git utility package?
+    """
+    args = ['git', 'show', os.path.join('HEAD:.', file_within_repo)]

[Stephen White, 2013/07/17 10:44:40]

Same here. I think the plumbing equivalent is "git cat-file -p", but I haven't
messed with it much.

[epoger, 2013/07/18 15:00:04]

I tried "git cat-file" in various permutations, but I couldn't get it to work.

Looking at
http://stackoverflow.com/questions/610208/how-to-retrieve-a-single-file-from-...
, I think I have two problems:
1. I need the full path to the file from git root, NOT the path from current
working dir
2. I need an object ID for the file in revision HEAD, which it seems I need to
get from another command

Given that "git show HEAD:./$RELATIVEPATH" works fine and keeps it simple, I'd
rather stick with that if it's OK with you.

[Stephen White, 2013/07/18 17:32:38]

Sure. Maybe just add a FIXME that we should try to avoid this porcelain command
somehow in the future.

[epoger, 2013/07/18 17:40:35]

Good idea... done.

+    with open(dest_path, 'wb') as outfile:
+        proc = subprocess.Popen(args, stdout=outfile)
+        proc.communicate()
+        if proc.returncode is not 0:
+            raise Exception('command "%s" failed' % args)
+
 def SvnDiff(path_to_skdiff, dest_dir, source_dir):
     """Generates a visual diff of all pending changes in source_dir.
 
     @param path_to_skdiff
     @param dest_dir existing directory within which to write results
     @param source_dir
     """
     # Validate parameters, filling in default values if necessary and possible.
     path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff))
     if not dest_dir:
         dest_dir = tempfile.mkdtemp()
     dest_dir = os.path.abspath(dest_dir)
 
     os.chdir(source_dir)
+    using_svn = os.path.isdir('.svn')
 
     # Prepare temporary directories.
     modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened')
     original_flattened_dir = os.path.join(dest_dir, 'original_flattened')
     diff_dir = os.path.join(dest_dir, 'diffs')
     for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] :
         shutil.rmtree(dir, ignore_errors=True)
         os.mkdir(dir)
 
     # Get a list of all locally modified (including added/deleted) files,
     # descending subdirectories.
-    svn_repo = svn.Svn('.')
-    modified_file_paths = svn_repo.GetFilesWithStatus(
-        svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
+    if using_svn:
+        svn_repo = svn.Svn('.')
+        modified_file_paths = svn_repo.GetFilesWithStatus(
+            svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
+    else:
+        modified_file_paths = _GitGetModifiedFiles()
 
     # For each modified file:
     # 1. copy its current contents into modified_flattened_dir
     # 2. copy its original contents into original_flattened_dir
     for modified_file_path in modified_file_paths:
         if modified_file_path.endswith('.json'):
             # Special handling for JSON files, in the hopes that they
             # contain GM result summaries.
             (_unused, original_file_path) = tempfile.mkstemp()
-            svn_repo.ExportBaseVersionOfFile(modified_file_path,
-                                             original_file_path)
+            if using_svn:
+                svn_repo.ExportBaseVersionOfFile(
+                    modified_file_path, original_file_path)
+            else:
+                _GitExportBaseVersionOfFile(
+                    modified_file_path, original_file_path)
             platform_prefix = re.sub(os.sep, '__',
                                      os.path.dirname(modified_file_path)) + '__'
             _CallJsonDiff(old_json_path=original_file_path,
                           new_json_path=modified_file_path,
                           old_flattened_dir=original_flattened_dir,
                           new_flattened_dir=modified_flattened_dir,
                           filename_prefix=platform_prefix)
             os.remove(original_file_path)
         else:
             dest_filename = re.sub(os.sep, '__', modified_file_path)
             # If the file had STATUS_DELETED, it won't exist anymore...
             if os.path.isfile(modified_file_path):
                 shutil.copyfile(modified_file_path,
                                 os.path.join(modified_flattened_dir,
                                              dest_filename))
-            svn_repo.ExportBaseVersionOfFile(
-                modified_file_path,
-                os.path.join(original_flattened_dir, dest_filename))
+            if using_svn:
+                svn_repo.ExportBaseVersionOfFile(
+                    modified_file_path,
+                    os.path.join(original_flattened_dir, dest_filename))
+            else:
+                _GitExportBaseVersionOfFile(
+                    modified_file_path,
+                    os.path.join(original_flattened_dir, dest_filename))
 
     # Run skdiff: compare original_flattened_dir against modified_flattened_dir
     RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir,
                                 modified_flattened_dir, diff_dir))
     print '\nskdiff results are ready in file://%s/index.html' % diff_dir
 
 def RaiseUsageException():
     raise Exception('%s\nRun with --help for more detail.' % (
         USAGE_STRING % __file__))
 
@@ skipping 17 matching lines @@
                       action='store', type='string', default=None,
                       help='path to already-built skdiff tool; if not set, '
                       'will search for it in typical directories near this '
                       'script')
     parser.add_option(OPTION_SOURCE_DIR,
                       action='store', type='string', default='.',
                       help='root directory within which to compare all ' +
                       'files; defaults to "%default"')
     (options, args) = parser.parse_args()
     Main(options, args)