Roll 50: Updated for push to origin/master.

tools/dom/scripts/idlsync.py

-#!/usr/bin/python
-# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
+# Upgrading Dart's SDK for HTML (blink IDLs).
+#
+# Typically this is done using the Dart integration branch (as it has to be
+# staged to get most things working).
+#
+# Enlist in third_party/WebCore:
+#      > cd src/dart/third_party
+#      > rm -rf WebCore    (NOTE: Normally detached head using gclient sync)
+#      > git clone https://github.com/dart-lang/webcore.git WebCore
+#
+# To update all *.idl, *.py, LICENSE files, and IDLExtendedAttributes.txt:
+#      > cd src/dart
+#      > python tools/dom/scripts/idlsync.py
+#
+# Display blink files to delete, copy, update, and collisions to review:
+#      > python tools/dom/scripts/idlsync.py --check
+#
+# Bring over all blink files to dart/third_party/WebCore (*.py, *.idl, and
+# IDLExtendedAttributes.txt):
+#      > python tools/dom/scripts/idlsync.py
+#
+# Update the DEPS file SHA for "WebCore_rev" with the committed changes of files
+# in WebCore e.g.,   "WebCore_rev": "@NNNNNNNNNNNNNNNNNNNNNNNNN"
+#
+# Generate the sdk/*.dart files from the new IDLs and PYTHON IDL parsing code
+# copied to in dart/third_party/WebCore from src/third_party/WebKit (blink).
+#
+#      > cd src/dart/tools/dom/script
+#      > ./go.sh
+#
+# Finally, commit the files in dart/third_party/WebCore.
 
 import optparse
-import os
 import os.path
 import re
 import requests
-import shutil
 import subprocess
 import sys
-import tempfile
+import time
 
-SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
-DART_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, '..', '..', '..'))
+from shutil import copyfile
 
 # Dartium DEPS file from the DEPS file checked into the dart-lang/sdk integration
 # branch.
-DEPS_GIT = ('https://raw.githubusercontent.com/dart-lang/sdk/'
-            'integration/tools/deps/dartium.deps/DEPS')
+DEPS_GIT = ('https://raw.githubusercontent.com/dart-lang/sdk/integration/'
+            'tools/deps/dartium.deps/DEPS')
+CHROME_TRUNK = "https://chromium.googlesource.com"
+WEBKIT_URL_PATTERN = r'"dartium_chromium_commit": "(\S+)",'
+DEPS_PATTERNS = {
+    'webkit': (CHROME_TRUNK, WEBKIT_URL_PATTERN),
+}
 
-# Whitelist of files to keep.
-WHITELIST = [
-    r'LICENSE(\S+)',
-    r'(\S+)\.idl',
-    r'(\S+)\.json',
-    r'(\S+)\.py',
-    r'(\S+)\.txt',
-    ]
+# Dartium/Chromium remote (GIT repository)
+GIT_REMOTES_CHROMIUM = 'https://chromium.googlesource.com/dart/dartium/src.git'
 
-# WebKit / WebCore info.
-CHROME_TRUNK = "https://src.chromium.org"
-WEBKIT_URL_PATTERN = r'"dartium_webkit_branch": "(\S+)",'
-WEBKIT_REV_PATTERN = r'"dartium_webkit_revision": "(\d+)",'
-WEBCORE_SUBPATH = 'Source/core'
-MODULES_SUBPATH = 'Source/modules'
-BINDINGS_SUBPATH = 'Source/bindings'
+# location of this file
+SOURCE_FILE_DIR = 'src/dart/tools/dom/scripts'
 
-LOCAL_WEBKIT_IDL_PATH = os.path.join(DART_PATH, 'third_party', 'WebCore')
-LOCAL_WEBKIT_README = """\
-This directory contains a copy of WebKit/WebCore IDL files.
-See the attached LICENSE-* files in this directory.
+WEBKIT_SOURCE = 'src/third_party/WebKit/Source'
+WEBCORE_SOURCE = 'src/dart/third_party/WebCore'
 
-Please do not modify the files here.  They are periodically copied
-using the script: $DART_ROOT/sdk/lib/html/scripts/%(script)s
+# Never automatically git add bindings/IDLExtendedAttributes.txt this file has
+# been modified by Dart but is usually changed by WebKit blink too.
+IDL_EXTENDED_ATTRIBUTES_FILE = 'IDLExtendedAttributes.txt'
 
-The current version corresponds to:
-URL: %(url)s
-Current revision: %(revision)s
-"""
+# Don't automatically update, delete or add anything in this directory:
+#      bindings/dart/scripts
+# The scripts in the above directory is the source for our Dart generators that
+# is driven from the blink IDL parser AST
+DART_SDK_GENERATOR_SCRIPTS = 'bindings/dart/scripts'
 
-# Chrome info.
-CHROME_URL_PATTERN = r'"dartium_chromium_branch": "(\S+)",'
-CHROME_REV_PATTERN = r'"dartium_chromium_revision": "(\d+)",'
-CHROME_IDL_SUBPATH = 'trunk/src/chrome/common/extensions/api'
-CHROME_COMMENT_EATER_SUBPATH = 'trunk/src/tools/json_comment_eater'
-CHROME_COMPILER_SUBPATH = 'trunk/src/tools/json_schema_compiler'
-CHROME_IDL_PARSER_SUBPATH = 'trunk/src/ppapi/generators'
-CHROME_PLY_SUBPATH = 'trunk/src/third_party/ply'
-LOCAL_CHROME_IDL_PATH = os.path.join(DART_PATH, 'third_party', 'chrome', 'idl')
-LOCAL_CHROME_COMMENT_EATER_PATH = os.path.join(
-      DART_PATH, 'third_party', 'chrome', 'tools', 'json_comment_eater')
-LOCAL_CHROME_COMPILER_PATH = os.path.join(DART_PATH, 'third_party', 'chrome',
-                                          'tools', 'json_schema_compiler')
-LOCAL_CHROME_IDL_PARSER_PATH = os.path.join(DART_PATH, 'third_party', 'chrome',
-                                            'ppapi', 'generators')
-LOCAL_CHROME_PLY_PATH = os.path.join(DART_PATH, 'third_party', 'chrome',
-                                     'third_party', 'ply')
-LOCAL_CHROME_README = """\
-This directory contains a copy of Chromium IDL and generation scripts
-used to generate Dart APIs for Chrome Apps.
+# sub directories containing IDLs (core and modules) from the base directory
+# src/third_party/WebKit/Source
+SUBDIRS = [
+    'bindings',
+    'core',
+    'modules',
+]
+IDL_EXT = '.idl'
+PY_EXT = '.py'
+LICENSE_FILE_PREFIX = 'LICENSE'    # e.g., LICENSE-APPLE, etc.
 
-The original files are from:
-URL: %(url)s
-Current revision: %(revision)s
+# Look in any file in WebCore we copy from WebKit if this comment is in the file
+# then flag this as a special .py or .idl file that needs to be looked at.
+DART_CHANGES = ' FIXMEDART: '
 
-Please see the corresponding LICENSE file at
-%(url)s/trunk/src/LICENSE.
-"""
-DEPTH_FILES = 'files'
-DEPTH_INFINITY = 'infinity'
+# application options passed in.
+options = None
 
-# Regular expressions corresponding to URL/revision patters in the
-# DEPS file.
-DEPS_PATTERNS = {
-    'webkit': (CHROME_TRUNK, WEBKIT_URL_PATTERN, WEBKIT_REV_PATTERN),
-#    'chrome': (CHROME_TRUNK, CHROME_URL_PATTERN, CHROME_REV_PATTERN),
-    }
+warning_messages = []
 
-# List of components to update.
-UPDATE_LIST = [
-    # (component, remote subpath, local path, local readme file, depth)
+# Is --check passed in.
+def isChecked():
+  global options
+  return options['check'] is not None
 
-    # WebKit IDL.
-    ('webkit', WEBCORE_SUBPATH, os.path.join(LOCAL_WEBKIT_IDL_PATH, 'core'),
-     LOCAL_WEBKIT_README, DEPTH_INFINITY),
-    ('webkit', MODULES_SUBPATH, os.path.join(LOCAL_WEBKIT_IDL_PATH, 'modules'),
-     LOCAL_WEBKIT_README, DEPTH_INFINITY),
-    ('webkit', BINDINGS_SUBPATH, os.path.join(LOCAL_WEBKIT_IDL_PATH, 'bindings'),
-     LOCAL_WEBKIT_README, DEPTH_INFINITY),
-
-    # Chrome IDL.
-    ('chrome', CHROME_IDL_SUBPATH, LOCAL_CHROME_IDL_PATH, LOCAL_CHROME_README,
-     DEPTH_INFINITY),
-    # Chrome PPAPI generators. Contains idl_parser.py which is used by the
-    # Chrome IDL compiler.
-    ('chrome', CHROME_IDL_PARSER_SUBPATH, LOCAL_CHROME_IDL_PARSER_PATH,
-     LOCAL_CHROME_README, DEPTH_FILES),
-    # ply files.
-    ('chrome', CHROME_PLY_SUBPATH, LOCAL_CHROME_PLY_PATH, LOCAL_CHROME_README,
-     DEPTH_INFINITY),
-    # Path for json_comment_eater, which is needed by the Chrome IDL compiler.
-    ('chrome', CHROME_COMMENT_EATER_SUBPATH, LOCAL_CHROME_COMMENT_EATER_PATH,
-     LOCAL_CHROME_README, DEPTH_FILES),
-    # Chrome IDL compiler files.
-    ('chrome', CHROME_COMPILER_SUBPATH, LOCAL_CHROME_COMPILER_PATH,
-     LOCAL_CHROME_README, DEPTH_INFINITY),
-    ]
-
+# Is --verbose passed in.
+def isVerbose():
+  global options
+  return options['verbose'] is not None
 
 def RunCommand(cmd, valid_exits=[0]):
   """Executes a shell command and return its stdout."""
-  print ' '.join(cmd)
+  if isVerbose():
+    print ' '.join(cmd)
   pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
   output = pipe.communicate()
   if pipe.returncode in valid_exits:
     return output[0]
   else:
     print output[1]
     print 'FAILED. RET_CODE=%d' % pipe.returncode
     sys.exit(pipe.returncode)
 
+# returns True if // FIXMEDART: is in the file.
+def anyDartFixMe(filepath):
+  if os.path.exists(filepath):
+    data = open(filepath, 'r').read()
+    return data.find(DART_CHANGES) != -1
+  else:
+    return False
+
+# Give a base_dir compute the trailing directory after base_dir
+# returns the subpath from base_dir for the path passed in.
+def subpath(path, base_dir):
+  dir_portion = ''
+  head = path
+  while True:
+    head, tail = os.path.split(head)
+    dir_portion = os.path.join(tail, dir_portion)
+    if head == base_dir or tail == '':
+      break;
+  return dir_portion
+
+# Copy any file in source_dir (WebKit) to destination_dir (dart/third_party/WebCore)
+# source_dir is the src/third_party/WebKit/Source location (blink)
+# destination_dir is the src/dart/third_party/WebCore location
+# returns idls_copied, py_copied, other_copied
+def copy_files(source_dir, destination_dir):
+  original_cwd = os.getcwd()
+  os.chdir(destination_dir)
+
+  idls = 0                  # *.idl files copied
+  pys = 0                   # *.py files copied
+  others = 0                # all other files copied
+
+  for (root, _, files) in os.walk(source_dir, topdown=False):
+    dir_portion = subpath(root, source_dir)
+    for f in files:
+      # Never automatically add any Dart generator scripts (these are the original
+      # sources in WebCore) from WebKit to WebCore.
+      if dir_portion != DART_SDK_GENERATOR_SCRIPTS:
+        if (f.endswith(IDL_EXT) or
+            f == IDL_EXTENDED_ATTRIBUTES_FILE or
+            f.endswith(PY_EXT) or
+            f.startswith(LICENSE_FILE_PREFIX)):
+          if f.endswith(IDL_EXT):
+            idls += 1
+          elif f.endswith(PY_EXT):
+            pys += 1
+          else:
+            others += 1
+          src_file = os.path.join(root, f)
+          dst_root = root.replace(WEBKIT_SOURCE, WEBCORE_SOURCE)
+          dst_file = os.path.join(dst_root, f)
+
+          destination = os.path.dirname(dst_file)
+          if not os.path.exists(destination):
+            os.makedirs(destination)
+
+          has_Dart_fix_me = anyDartFixMe(dst_file)
+
+          if not isChecked():
+            copyfile(src_file, dst_file)
+          if isVerbose():
+            print('...copying %s' % os.path.split(dst_file)[1])
+          if f == IDL_EXTENDED_ATTRIBUTES_FILE:
+            warning_messages.append(dst_file)
+          else:
+            if has_Dart_fix_me:
+              warning_messages.append(dst_file)
+            if not (isChecked() or has_Dart_fix_me):
+              # git add the file
+              RunCommand(['git', 'add', dst_file])
+
+  os.chdir(original_cwd)
+
+  return [idls, pys, others]
+
+# Remove any file in webcore_dir that no longer exist in the webkit_dir
+# webcore_dir src/dart/third_party/WebCore location
+# webkit_dir src/third_party/WebKit/Source location (blink)
+# only check if the subdir off from webcore_dir
+# return list of files deleted
+def remove_obsolete_webcore_files(webcore_dir, webkit_dir, subdir):
+  files_to_delete = []
+
+  original_cwd = os.getcwd()
+  os.chdir(webcore_dir)
+
+  for (root, _, files) in os.walk(os.path.join(webcore_dir, subdir), topdown=False):
+    dir_portion = subpath(root, webcore_dir)
+    for f in files:
+      # Never automatically deleted any Dart generator scripts (these are the
+      # original sources in WebCore).
+      if dir_portion != DART_SDK_GENERATOR_SCRIPTS:
+        check_file = os.path.join(dir_portion, f)
+        check_file_full_path = os.path.join(webkit_dir, check_file)
+        if not os.path.exists(check_file_full_path):
+          if not isChecked():
+            # Remove the file using git
+            RunCommand(['git', 'rm', check_file])
+          files_to_delete.append(check_file)
+
+  os.chdir(original_cwd)
+
+  return files_to_delete
+
+def ParseOptions():
+  parser = optparse.OptionParser()
+  parser.add_option('--verbose', '-v', dest='verbose', action='store_false',
+                    help='Dump all information', default=None)
+  parser.add_option('--check', '-c', dest='check', action='store_false',
+                    help='Display results without adding, updating or deleting any files', default=None)
+  args, _ = parser.parse_args()
+
+  argOptions = {}
+  argOptions['verbose'] = args.verbose
+  argOptions['check'] = args.check
+  return argOptions
+
+# Fetch the DEPS file in src/dart/tools/deps/dartium.deps/DEPS from the GIT repro.
 def GetDepsFromGit():
   req = requests.get(DEPS_GIT)
   return req.text
 
-def GetSvnRevision(deps, component):
-  """Returns a tuple with the (dartium webkit repo, latest revision)."""
-  url_base, url_pattern, rev_pattern = DEPS_PATTERNS[component]
+def ValidateGitRemotes():
+  #origin  https://chromium.googlesource.com/dart/dartium/src.git (fetch)
+  remotes_list = RunCommand(['git', 'remote', '--verbose']).split()
+  if (len(remotes_list) > 2 and
+      remotes_list[0] == 'origin' and remotes_list[1] == GIT_REMOTES_CHROMIUM):
+    return True
+
+  print 'ERROR: Unable to find dart/dartium/src repository %s' % GIT_REMOTES_CHROMIUM
+  return False
+
+def getCurrentDartiumSHA():
+  cwd = os.getcwd()
+  if cwd.endswith('dart'):
+    # In src/dart 
+    src_dir, _ = os.path.split(cwd)
+  elif cwd.endswith('src'):
+    src_dir = cwd
+  else:
+    src_dir = os.path.join(cwd, 'src')
+  os.chdir(src_dir)
+
+  if ValidateGitRemotes():
+    dartium_sha = RunCommand(['git', 'log', '--format=format:%H', '-1'])
+  else:
+    dartium_sha = -1
+
+  os.chdir(cwd)
+  return dartium_sha
+
+# Returns the SHA of the Dartium/Chromiun in the DEPS file.
+def GetDEPSDartiumGitRevision(deps, component):
+  """Returns a tuple with the (dartium chromium repo, latest revision)."""
+  url_base, url_pattern = DEPS_PATTERNS[component]
   url = url_base + re.search(url_pattern, deps).group(1)
-  revision = re.search(rev_pattern, deps).group(1)
-  return (url, revision)
-
-
-def RefreshFiles(url, revision, remote_path, local_path, depth):
-  """Refreshes refreshes files in the local_path to specific url /
-  revision / remote_path, exporting to depth"""
-  cwd = os.getcwd()
-  try:
-    if os.path.exists(local_path):
-      shutil.rmtree(local_path)
-    head, tail = os.path.split(local_path)
-    if not os.path.exists(head):
-      os.makedirs(head)
-    os.chdir(head)
-    RunCommand(['svn', 'export', '--depth', depth, '-r', revision,
-                url + '/' + remote_path, tail])
-  finally:
-    os.chdir(cwd)
-
-
-def PruneExtraFiles(local_path):
-  """Removes all files that do not match the whitelist."""
-  pattern = re.compile(reduce(lambda x,y: '%s|%s' % (x,y),
-                              map(lambda z: '(%s)' % z, WHITELIST)))
-  for (root, dirs, files) in os.walk(local_path, topdown=False):
-    for f in files:
-      if not pattern.match(f):
-        os.remove(os.path.join(root, f))
-    for d in dirs:
-      dirpath = os.path.join(root, d)
-      if not os.listdir(dirpath):
-        shutil.rmtree(dirpath)
-
-
-def GenerateReadme(local_path, template, url, revision):
-  readme = template % {
-    'script': os.path.basename(__file__),
-    'url': url,
-    'revision': revision }
-
-  readme_path = os.path.join(local_path, 'README')
-  out = open(readme_path, 'w')
-  out.write(readme)
-  out.close()
-
-ZIP_ARCHIVE = 'version-control-dirs.zip'
-
-def SaveVersionControlDir(local_path):
-  if os.path.exists(local_path):
-    RunCommand([
-      'sh', '-c',
-      'find %s -name .svn -or -name .git | zip -r %s -@' % (
-          os.path.relpath(local_path), ZIP_ARCHIVE)
-    ], [0, 12])  # It is ok if zip has nothing to do (exit code 12).
-
-def RestoreVersionControlDir(local_path):
-  archive_path = os.path.join(local_path, ZIP_ARCHIVE)
-  if os.path.exists(archive_path):
-    RunCommand(['unzip', ZIP_ARCHIVE, '-d', '.'])
-    RunCommand(['rm', ZIP_ARCHIVE])
-
-def ParseOptions():
-  parser = optparse.OptionParser()
-  parser.add_option('--webkit-revision', '-w', dest='webkit_revision',
-                    help='WebKit IDL revision to install', default=None)
-  parser.add_option('--chrome-revision', '-c', dest='chrome_revision',
-                    help='Chrome IDL revision to install', default=None)
-  args, _ = parser.parse_args()
-  update = {}
-  update['webkit'] = args.webkit_revision
-  return update
-
+  # Get the SHA for the Chromium/WebKit changes for Dartium.
+  revision = url[len(url_base):]
+  return revision
 
 def main():
-  update = ParseOptions()
+  global options
+  options = ParseOptions()
+
+  current_dir = os.path.dirname(os.path.abspath(__file__))
+  if not current_dir.endswith(SOURCE_FILE_DIR):
+    print 'ERROR: idlsync.py not run in proper directory (%s)\n', current_dir
+
+  base_directory = current_dir[:current_dir.rfind(SOURCE_FILE_DIR)]
+
+  # Validate that the DEPS SHA matches the SHA of the chromium/dartium branch.
   deps = GetDepsFromGit()
-  for (component, remote_path, local_path, readme, depth) in UPDATE_LIST:
-    if component in update.keys():
-      revision = update[component]
-      url, latest = GetSvnRevision(deps, component)
-      if revision is None:
-        revision = latest
-      SaveVersionControlDir(local_path);
-      RefreshFiles(url, revision, remote_path, local_path, depth)
-      PruneExtraFiles(local_path)
-      GenerateReadme(local_path, readme, url, revision)
-      RestoreVersionControlDir(local_path);
+  revision = GetDEPSDartiumGitRevision(deps, 'webkit')
+  dartium_sha = getCurrentDartiumSHA()
+  if not(revision == dartium_sha):
+    print "ERROR: Chromium/Dartium SHA in DEPS doesn't match the GIT branch."
+    return
+
+  start_time = time.time()
+  for subdir in SUBDIRS:
+    webkit_dir = os.path.join(base_directory, WEBKIT_SOURCE) 
+    webcore_dir = os.path.join(base_directory, WEBCORE_SOURCE)
+
+    idls_deleted = remove_obsolete_webcore_files(webcore_dir, webkit_dir, subdir)
+    print "%s files removed in WebCore %s" % (idls_deleted.__len__(), subdir)
+    if isVerbose():
+      for delete_file in idls_deleted:
+        print "    %s" % delete_file
+
+    idls_copied, py_copied, other_copied = copy_files(os.path.join(webkit_dir, subdir), webcore_dir)
+    print "Copied %s IDLs to %s" % (idls_copied, subdir)
+    print "Copied %s PYs to %s" % (py_copied, subdir)
+    print "Copied %s other to %s\n" % (other_copied, subdir)
+
+  end_time = time.time()
+
+  print 'WARNING: File(s) contain FIXMEDART and are NOT "git add " please review:'
+  for warning in warning_messages:
+    print '    %s' % warning
+
+  print '\nDone idlsync completed in %s seconds' % round(end_time - start_time, 2)
 
 if __name__ == '__main__':
-  main()
+  sys.exit(main())