Deterministic build: Add a whitelist to the compare_build_artifact script

scripts/slave/recipe_modules/isolate/resources/compare_build_artifacts.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Compare the artifacts from two builds."""
 
 import difflib
 import json
 import optparse
 import os
 import struct
 import sys
 import time
 
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+# List of files that are known to be non deterministic. This is a temporary
+# workaround to find regression on the deterministic builders.
+# TODO(sebmarchand): Remove this once all the files are deterministic.
+WHITELIST = {
+  'win': {
+    'base_unittests.exe',
+    'base_unittests.isolated',
+    'browser_tests.exe',
+    'browser_tests.isolated',
+    'chrome.dll',
+    'chrome.exe',
+    'chrome_child.dll',
+    'chrome_watcher.dll',
+    'clearkeycdm.dll',
+    'content_browsertests.exe',
+    'content_browsertests.isolated',
+    'content_unittests.exe',
+    'content_unittests.isolated',
+    'd8.exe',
+    'delegate_execute.exe',
+    'delegate_execute_unittests.exe',
+    'interactive_ui_tests.exe',
+    'interactive_ui_tests.isolated',
+    'metro_driver.dll',
+    'mock_nacl_gdb.exe',
+    'net_unittests.exe',
+    'net_unittests.isolated',
+    'npapi_test_plugin.dll',
+    'pdf.dll',
+    'peerconnection_server.exe',
+    'ppapi_nacl_tests_pnacl_newlib_x32.nexe',
+    'ppapi_nacl_tests_pnacl_newlib_x64.nexe',
+    'sync_integration_tests.exe',
+    'sync_integration_tests.isolated',
+    'test_registrar.exe',
+    'unit_tests.exe',
+    'unit_tests.isolated',
+  },
+  'linux': {
+    'browser_tests.isolated',
+    'nacl_helper_nonsfi',
+    'nacl_irt_x86_64.nexe',
+    'ppapi_nacl_tests_glibc_x64.nexe',
+    'ppapi_nacl_tests_newlib_x64.nexe',
+    'ppapi_nacl_tests_pnacl_newlib_x64.nexe',
+    'unit_tests.isolated',
+  },
+  'mac': { },
+  'android': { },
+  'ios': { },
+}
+
 def get_files_to_compare(build_dir, recursive=False):
   """Get the list of files to compare."""
   allowed = frozenset(
       ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so'))
   non_x_ok_exts = frozenset(('.apk', '.isolated'))
   def check(f):
     if not os.path.isfile(f):
       return False
     if os.path.basename(f).startswith('.'):
       return False
@@ skipping 87 matching lines @@
     # else, falls through binary comparison, it must be binary equal too.
 
   file_len = os.stat(first_filepath).st_size
   if file_len != os.stat(second_filepath).st_size:
     return 'different size: %d != %d' % (
         file_len, os.stat(second_filepath).st_size)
 
   return diff_binary(first_filepath, second_filepath, file_len)
 
 
-def compare_build_artifacts(first_dir, second_dir, recursive=False):
+def compare_build_artifacts(first_dir, second_dir, target_platform,
+                            recursive=False):
   """Compare the artifacts from two distinct builds."""
   if not os.path.isdir(first_dir):
     print >> sys.stderr, '%s isn\'t a valid directory.' % first_dir
     return 1
   if not os.path.isdir(second_dir):
     print >> sys.stderr, '%s isn\'t a valid directory.' % second_dir
     return 1
 
   with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f:
     blacklist = frozenset(json.load(f))
 
-  res = 0
+  whitelist = WHITELIST[target_platform]
+
+  unexpected_failures = 0
+  expected_failures = 0
   first_list = get_files_to_compare(first_dir, recursive) - blacklist
   second_list = get_files_to_compare(second_dir, recursive) - blacklist
 
   diff = first_list.symmetric_difference(second_list)
   if diff:
     print >> sys.stderr, 'Different list of files in both directories'
     print >> sys.stderr, '\n'.join('  ' + i for i in sorted(diff))
-    res += len(diff)
+    unexpected_failures += len(diff)
 
   epoch_hex = struct.pack('<I', int(time.time())).encode('hex')
   print('Epoch: %s' %
       ' '.join(epoch_hex[i:i+2] for i in xrange(0, len(epoch_hex), 2)))
   max_filepath_len = max(len(n) for n in first_list & second_list)
   for f in sorted(first_list & second_list):
     first_file = os.path.join(first_dir, f)
     second_file = os.path.join(second_dir, f)
     result = compare_files(first_file, second_file)
     if not result:
       result = 'equal'
     else:
-      result = 'DIFFERENT: %s' % result
-      res += 1
+      expected = 'unexpected'
+      if f in whitelist:
+        expected_failures += 1
+        expected = 'expected'
+      else:
+        unexpected_failures += 1
+      result = 'DIFFERENT (%s): %s' % (expected, result)
     print('%-*s: %s' % (max_filepath_len, f, result))
 
   print '%d files are equal, %d are different.'  % (
-      len(first_list & second_list) - res, res)
+      len(first_list & second_list) - expected_failures - unexpected_failures,
+      expected_failures + unexpected_failures)
 
-  return 0 if res == 0 else 1
+  return 0 if unexpected_failures == 0 else 1
 
 
 def main():
   parser = optparse.OptionParser(usage='%prog [options]')
   parser.add_option(
       '-f', '--first-build-dir', help='The first build directory.')
   parser.add_option(
       '-s', '--second-build-dir', help='The second build directory.')
   parser.add_option('-r', '--recursive', action='store_true', default=False,
                     help='Indicates if the comparison should be recursive.')
+  parser.add_option('-t', '--target-platform', help='The target platform.')
   options, _ = parser.parse_args()
 
   if not options.first_build_dir:
     parser.error('--first-build-dir is required')
   if not options.second_build_dir:
     parser.error('--second-build-dir is required')
+  if not options.target_platform:
+    parser.error('--target-platform is required')
 
   return compare_build_artifacts(os.path.abspath(options.first_build_dir),
                                  os.path.abspath(options.second_build_dir),
+                                 options.target_platform,
                                  options.recursive)
 
 
 if __name__ == '__main__':
   sys.exit(main())