[android_webview] Deps whitelist and scripts for AOSP builder

android_webview/buildbot/deps_whitelist.py

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+"""Logic to generate lists of DEPS used by various parts of
+the android_webview continuous integration (buildbot) infrastructure.
+
+Note: The root Chromium project (which is not explicitly listed here)
+has a couple of third_party libraries checked in directly into it. This means
+that the list of third parties present in this file is not a comprehensive
+list of third party android_webview dependencies.
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+
+
+class DepsWhitelist(object):
+  def __init__(self):
+    # Dependencies required to build android_webview.
+    self._compile_dependencies = [
+      'googleurl',
+      'sdch/open-vcdiff',
+      'testing/gtest',
+      'third_party/WebKit',
+      'third_party/angle',
+      ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
+       'braille'),
+      'third_party/freetype',
+      'third_party/icu',
+      'third_party/leveldatabase/src',
+      'third_party/libjingle/source',
+      'third_party/libphonenumber/src/phonenumbers',
+      'third_party/libphonenumber/src/resources',
+      'third_party/openssl',
+      'third_party/opus/src',
+      'third_party/ots',
+      'third_party/skia/gyp',
+      'third_party/skia/include',
+      'third_party/skia/src',
+      'third_party/smhasher/src',
+      'third_party/v8-i18n',
+      'third_party/yasm/source/patched-yasm',
+      'tools/grit',
+      'tools/gyp',
+      'v8',
+    ]
+
+    # Dependencies that need to be merged into the Android tree.
+    self._snapshot_into_android_dependencies = self._compile_dependencies
+
+    # Dependencies required to run android_webview tests but not required to
+    # compile.
+    self._test_data_dependencies = [
+      'chrome/test/data/perf/third_party/octane',
+    ]
+
+  @staticmethod
+  def _read_deps_file(deps_file_path):
+    class FileImplStub(object):
+      """Stub for the File syntax."""
+      def __init__(self, file_location):
+        pass
+
+      @staticmethod
+      def GetPath():
+        return ''
+
+      @staticmethod
+      def GetFilename():
+        return ''
+
+      @staticmethod
+      def GetRevision():
+        return None
+
+    def from_stub(__, _=None):
+      """Stub for the From syntax."""
+      return ''
+
+    class VarImpl(object):
+      def __init__(self, custom_vars, local_scope):
+        self._custom_vars = custom_vars
+        self._local_scope = local_scope
+
+      def Lookup(self, var_name):
+        """Implements the Var syntax."""
+        if var_name in self._custom_vars:
+          return self._custom_vars[var_name]
+        elif var_name in self._local_scope.get("vars", {}):
+          return self._local_scope["vars"][var_name]
+        raise Exception("Var is not defined: %s" % var_name)
+
+    local_scope = {}
+    var = VarImpl({}, local_scope)
+    global_scope = {
+        'File': FileImplStub,
+        'From': from_stub,
+        'Var': var.Lookup,
+        'deps_os': {},
+    }
+    execfile(deps_file_path, global_scope, local_scope)
+    deps = local_scope.get('deps', {})
+    deps_os = local_scope.get('deps_os', {})
+    for os_specific_deps in deps_os.itervalues():
+      deps.update(os_specific_deps)
+    return deps.keys()
+
+  def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
+    """Calculates the list of deps that need to be excluded from the deps_file
+    so that the only deps left are the one in the whitelist."""
+    all_deps = self._read_deps_file(deps_file_path)
+    # The list of deps read from the DEPS file are prefixed with the source
+    # tree root, which is 'src' for Chromium.
+    def prepend_root(path):
+      return os.path.join('src', path)
+    whitelisted_deps = map(prepend_root, whitelisted_deps)
+    deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
+    return dict(map(lambda(x): (x, None), deps_blacklist))
+
+  def get_deps_for_android_build(self, deps_file_path):
+    """This is used to calculate the custom_deps list for the Android bot.
+    """
+    if not deps_file_path:
+      raise Exception('You need to specify a DEPS file path.')
+    return self._make_gclient_blacklist(deps_file_path,
+                                        self._compile_dependencies)
+
+  def get_deps_for_android_build_and_test(self, deps_file_path):
+    """This is used to calculate the custom_deps list for the Android perf bot.
+    """
+    if not deps_file_path:
+      raise Exception('You need to specify a DEPS file path.')
+    return self._make_gclient_blacklist(deps_file_path,
+                                        self._compile_dependencies +
+                                        self._test_data_dependencies)
+
+  def get_deps_for_android_merge(self, _):
+    """Calculates the list of deps that need to be merged into the Android tree
+    in order to build the C++ and Java android_webview code."""
+    return self._snapshot_into_android_dependencies
+
+  def get_deps_for_license_check(self, _):
+    """Calculates the list of deps that need to be checked for Android license
+    compatibility"""
+    return self._snapshot_into_android_dependencies
+
+  def execute_method(self, method_name, deps_file_path):
+    methods = {
+      'android_build': self.get_deps_for_android_build,
+      'android_build_and_test':
+        self.get_deps_for_android_build_and_test,
+      'android_merge': self.get_deps_for_android_merge,
+      'license_check': self.get_deps_for_license_check
+    }
+    if not method_name in methods:
+      raise Exception('Method name %s is not valid. Valid choices are %s' %
+                      (method_name, methods.keys()))
+    return methods[method_name](deps_file_path)
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--method', help='Method to use to fetch from whitelist.',
+                      required=True)
+  parser.add_argument('--path-to-deps', help='Path to DEPS file.')
+  parser.add_argument('--output-json', help='Name of file to write output to.')
+  parser.add_argument('verbose', action='store_true', default=False)
+  opts = parser.parse_args()
+
+  logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)
+
+  deps_whitelist = DepsWhitelist()
+  blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)
+
+  if (opts.output_json):
+    output_dict = {
+        'blacklist' : blacklist
+    }
+    with open(opts.output_json, 'w') as output_json_file:
+      json.dump(output_dict, output_json_file)
+  else:
+    print blacklist
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())