Add known_incompatible.py items to the blacklist in deps_whitelist.py

android_webview/buildbot/deps_whitelist.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
 """Logic to generate lists of DEPS used by various parts of
 the android_webview continuous integration (buildbot) infrastructure.
 
 Note: The root Chromium project (which is not explicitly listed here)
 has a couple of third_party libraries checked in directly into it. This means
 that the list of third parties present in this file is not a comprehensive
 list of third party android_webview dependencies.
 """
 
 import argparse
 import json
 import logging
 import os
 import sys
 
+# Add android_webview/tools to path to get at known_issues.
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools'))
+import known_issues
+
 
 class DepsWhitelist(object):
   def __init__(self):
     # If a new DEPS entry is needed for the AOSP bot to compile please add it
     # here first.
     # This is a staging area for deps that are accepted by the android_webview
     # team and are in the process of having the required branches being created
     # in the Android tree.
     self._compile_but_not_snapshot_dependencies = [
     ]
@@ skipping 25 matching lines @@
       'third_party/skia',
       'third_party/smhasher/src',
       'third_party/usrsctp/usrsctplib',
       'third_party/webrtc',
       'third_party/yasm/source/patched-yasm',
       'tools/grit',
       'tools/gyp',
       'v8',
     ]
 
+    self._uneeded_for_building_android = [

[mkosiba (inactive), 2014/06/11 09:38:38]

maybe _prune_from_rsync_build ? And add a comment saying that this is only to
save time.

[hjd, 2014/06/11 09:59:25]

Done.

+        'third_party/WebKit/LayoutTests',
+    ]
+
     # Dependencies required to build android_webview.
     self._compile_dependencies = (self._snapshot_into_android_dependencies +
                                   self._compile_but_not_snapshot_dependencies)
 
     # Dependencies required to run android_webview tests but not required to
     # compile.
     self._test_data_dependencies = [
       'chrome/test/data/perf/third_party/octane',
     ]
 
@@ skipping 41 matching lines @@
         'Var': var.Lookup,
         'deps_os': {},
     }
     execfile(deps_file_path, global_scope, local_scope)
     deps = local_scope.get('deps', {})
     deps_os = local_scope.get('deps_os', {})
     for os_specific_deps in deps_os.itervalues():
       deps.update(os_specific_deps)
     return deps.keys()
 
+  def _get_known_issues(self):
+    issues = []
+    for root, paths in known_issues.KNOWN_INCOMPATIBLE.items():
+      for path in paths:
+        # The leading dot seems to confuse rsync.

[mkosiba (inactive), 2014/06/11 09:38:38]

maybe use os.path.normpath instead of a special case?

[hjd, 2014/06/11 09:59:25]

Done.

+        if root == '.':
+          issues.append(path)
+        else:
+          issues.append(os.path.join(root, path))
+    return issues
+
   def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
     """Calculates the list of deps that need to be excluded from the deps_file
     so that the only deps left are the one in the whitelist."""
     all_deps = self._read_deps_file(deps_file_path)
     # The list of deps read from the DEPS file are prefixed with the source
     # tree root, which is 'src' for Chromium.
     def prepend_root(path):
       return os.path.join('src', path)
     whitelisted_deps = map(prepend_root, whitelisted_deps)
     deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
     return dict(map(lambda(x): (x, None), deps_blacklist))
 
+  def _make_blacklist(self, deps_file_path, whitelisted_deps):
+    """Calculates the list of paths we should exclude """
+    all_deps = self._read_deps_file(deps_file_path)
+    def remove_src_prefix(path):
+      return path.replace('src/', '', 1)
+    all_deps = map(remove_src_prefix, all_deps)
+    # Ignore all deps except those whitelisted.
+    blacklist = set(all_deps).difference(whitelisted_deps)
+    # Ignore the 'known issues'. Typically these are the licence incompatible
+    # things checked directly into Chromium.
+    blacklist = blacklist.union(self._get_known_issues())
+    # Ignore any other non-deps, non-licence paths we don't like.
+    blacklist = blacklist.union(self._uneeded_for_building_android)
+    return list(blacklist)
+
   def get_deps_for_android_build(self, deps_file_path):
     """This is used to calculate the custom_deps list for the Android bot.
     """
     if not deps_file_path:
       raise Exception('You need to specify a DEPS file path.')
     return self._make_gclient_blacklist(deps_file_path,
                                         self._compile_dependencies)
 
   def get_deps_for_android_build_and_test(self, deps_file_path):
     """This is used to calculate the custom_deps list for the Android perf bot.
     """
     if not deps_file_path:
       raise Exception('You need to specify a DEPS file path.')
     return self._make_gclient_blacklist(deps_file_path,
                                         self._compile_dependencies +
                                         self._test_data_dependencies)
 
+  def get_blacklist_for_android_build(self, deps_file_path):
+    """Calculates the list of paths we should exclude when building Android
+    either because of license compatibility or because they are large and
+    uneeded.
+    """
+    if not deps_file_path:
+      raise Exception('You need to specify a DEPS file path.')
+    return self._make_blacklist(deps_file_path, self._compile_dependencies)
+
   def get_deps_for_android_merge(self, _):
     """Calculates the list of deps that need to be merged into the Android tree
     in order to build the C++ and Java android_webview code."""
     return self._snapshot_into_android_dependencies
 
   def get_deps_for_license_check(self, _):
     """Calculates the list of deps that need to be checked for Android license
     compatibility"""
     return self._compile_dependencies
 
+
   def execute_method(self, method_name, deps_file_path):
     methods = {
       'android_build': self.get_deps_for_android_build,
       'android_build_and_test':
         self.get_deps_for_android_build_and_test,
       'android_merge': self.get_deps_for_android_merge,
-      'license_check': self.get_deps_for_license_check
+      'license_check': self.get_deps_for_license_check,
+      'android_build_blacklist': self.get_blacklist_for_android_build,

[mkosiba (inactive), 2014/06/11 09:38:38]

maybe android_rsync_build ?

[hjd, 2014/06/11 09:59:25]

Done.

     }
     if not method_name in methods:
       raise Exception('Method name %s is not valid. Valid choices are %s' %
                       (method_name, methods.keys()))
     return methods[method_name](deps_file_path)
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--method', help='Method to use to fetch from whitelist.',
                       required=True)
@@ skipping 14 matching lines @@
     with open(opts.output_json, 'w') as output_json_file:
       json.dump(output_dict, output_json_file)
   else:
     print blacklist
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())