Show detailed difference on unexpected difference.

tools/determinism/compare_build_artifacts.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Compare the artifacts from two builds."""
 
 import difflib
 import json
 import optparse
 import os
+import re
+import subprocess
 import struct
 import sys
 import time
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 # List of files that are known to be non deterministic. This is a "temporary"
 # workaround to find regression on the deterministic builders.
 #
@@ skipping 466 matching lines @@
     # else, falls through binary comparison, it must be binary equal too.
 
   file_len = os.stat(first_filepath).st_size
   if file_len != os.stat(second_filepath).st_size:
     return 'different size: %d != %d' % (
         file_len, os.stat(second_filepath).st_size)
 
   return diff_binary(first_filepath, second_filepath, file_len)
 
 
+def get_deps(build_dir, target):
+  """Returns list of object files needed to build target."""
+  NODE_PATTERN = re.compile(r'label="([a-zA-Z0-9_\\/.-]+)"')
+  CHECK_EXTS = ('.o', '.obj')
+  try:
+    out = subprocess.check_output(['ninja', '-C', build_dir,

[M-A Ruel, 2016/08/24 18:47:27]

There's a problem as the directory was renamed, so calling ninja after the
directory was renamed will likely not result in the expected outcome, as the
ninja files may have absolute paths. So it is important to rename the directory
:/ before calling ninja.

[Yoshisato Yanagisawa, 2016/08/25 08:47:29]

I have never seen absolute object file path names but I am not confident.  Let
me rename to possibly original directory, and warn and skip if absolute object
file path is used.

[M-A Ruel, 2016/08/25 15:46:26]

zforman@ worked on making this work on linux but this is likely not the case on
OSX and Windows, probably worth double checking on at least one of the other
platform eventually.

[Yoshisato Yanagisawa, 2016/08/29 05:44:49]

I checked the code seems to work on Win.

+                                   '-t', 'graph', target])
+  except subprocess.CalledProcessError as e:
+    print >> sys.stderr, 'error to get graph for %s: %s' % (target, e)
+    return []
+  files = []
+  for line in out.splitlines():
+    matched = NODE_PATTERN.search(line)
+    if matched:
+      path = matched.group(1)
+      if not os.path.splitext(path)[1] in CHECK_EXTS:
+        continue
+      files.append(path)
+  return files
+
+
+def compare_deps(first_dir, second_dir, targets):
+  """Print difference of dependent files."""
+  for target in targets:
+    print 'Checking %s difference:' % target
+    first_deps = get_deps(first_dir, target)
+    second_deps = get_deps(second_dir, target)
+    deps = set(first_deps).union(set(second_deps))

[M-A Ruel, 2016/08/24 18:47:27]

the difference should be empty ?

[Yoshisato Yanagisawa, 2016/08/25 08:47:29]

I am not confident.  However, I will simplify the code expecting no difference
between first_deps and second_deps.

[M-A Ruel, 2016/08/25 15:46:26]

I guess my question was more to print out the differences to be able to monitor
when this happens.

[Yoshisato Yanagisawa, 2016/08/29 05:44:49]

I think current code still help us to find this kind of difference, not clearly
mentions which one is larger, though.
Let me update the code when we actually see this case.

+    for d in deps:
+      first_file = os.path.join(first_dir, d)
+      second_file = os.path.join(second_dir, d)
+      if not os.path.exists(first_file):
+        print '%s: not found' % first_file
+        continue
+      if not os.path.exists(second_file):
+        print '%s: not found' % second_file
+        continue
+      compare_files(first_file, second_file)
+
+
 def compare_build_artifacts(first_dir, second_dir, target_platform,
                             recursive=False):
   """Compares the artifacts from two distinct builds."""
   if not os.path.isdir(first_dir):
     print >> sys.stderr, '%s isn\'t a valid directory.' % first_dir
     return 1
   if not os.path.isdir(second_dir):
     print >> sys.stderr, '%s isn\'t a valid directory.' % second_dir
     return 1
 
@@ skipping 46 matching lines @@
   print('Unexpected diffs: %d' % len(unexpected_diffs))
   if unexpected_diffs:
     print('Unexpected files with diffs:\n')
     for u in unexpected_diffs:
       print('  %s' % u)
   if unexpected_equals:
     print('Unexpected files with no diffs:\n')
     for u in unexpected_equals:
       print('  %s' % u)
 
+  if unexpected_diffs:
+    diffs_to_investigate = set(unexpected_diffs).difference(
+        set(missing_files))
+    compare_deps(first_dir, second_dir, diffs_to_investigate)
+
   return int(bool(unexpected_diffs))
 
 
 def main():
   parser = optparse.OptionParser(usage='%prog [options]')
   parser.add_option(
       '-f', '--first-build-dir', help='The first build directory.')
   parser.add_option(
       '-s', '--second-build-dir', help='The second build directory.')
   parser.add_option('-r', '--recursive', action='store_true', default=False,
                     help='Indicates if the comparison should be recursive.')
   target = {
       'darwin': 'mac', 'linux2': 'linux', 'win32': 'win'
   }.get(sys.platform, sys.platform)
-  parser.add_option('-t', '--target-platform', help='The target platform.'
+  parser.add_option('-t', '--target-platform', help='The target platform.',
                     default=target, choices=('android', 'mac', 'linux', 'win'))
   options, _ = parser.parse_args()
 
   if not options.first_build_dir:
     parser.error('--first-build-dir is required')
   if not options.second_build_dir:
     parser.error('--second-build-dir is required')
   if not options.target_platform:
     parser.error('--target-platform is required')
 
   return compare_build_artifacts(os.path.abspath(options.first_build_dir),
                                  os.path.abspath(options.second_build_dir),
                                  options.target_platform,
                                  options.recursive)
 
 
 if __name__ == '__main__':
   sys.exit(main())