Index: tools/dom/scripts/idlsync.py |
diff --git a/tools/dom/scripts/idlsync.py b/tools/dom/scripts/idlsync.py |
index 24aae09cca20debca8b5ede0a60694436dd3114d..632e236e75aee017df0b20cc12808b6709d5fde3 100755 |
--- a/tools/dom/scripts/idlsync.py |
+++ b/tools/dom/scripts/idlsync.py |
@@ -1,128 +1,108 @@ |
-#!/usr/bin/python |
-# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
-# for details. All rights reserved. Use of this source code is governed by a |
-# BSD-style license that can be found in the LICENSE file. |
+# Upgrading Dart's SDK for HTML (blink IDLs). |
+# |
+# Typically this is done using the Dart integration branch (as it has to be |
+# staged to get most things working). |
+# |
+# Enlist in third_party/WebCore: |
+# > cd src/dart/third_party |
+# > rm -rf WebCore (NOTE: Normally detached head using gclient sync) |
+# > git clone https://github.com/dart-lang/webcore.git WebCore |
+# |
+# To update all *.idl, *.py, LICENSE files, and IDLExtendedAttributes.txt: |
+# > cd src/dart |
+# > python tools/dom/scripts/idlsync.py |
+# |
+# Display blink files to delete, copy, update, and collisions to review: |
+# > python tools/dom/scripts/idlsync.py --check |
+# |
+# Bring over all blink files to dart/third_party/WebCore (*.py, *.idl, and |
+# IDLExtendedAttributes.txt): |
+# > python tools/dom/scripts/idlsync.py |
+# |
+# Update the DEPS file SHA for "WebCore_rev" with the committed changes of files |
+# in WebCore e.g., "WebCore_rev": "@NNNNNNNNNNNNNNNNNNNNNNNNN" |
+# |
+# Generate the sdk/*.dart files from the new IDLs and PYTHON IDL parsing code |
+# copied to in dart/third_party/WebCore from src/third_party/WebKit (blink). |
+# |
+# > cd src/dart/tools/dom/script |
+# > ./go.sh |
+# |
+# Finally, commit the files in dart/third_party/WebCore. |
import optparse |
-import os |
import os.path |
import re |
import requests |
-import shutil |
import subprocess |
import sys |
-import tempfile |
+import time |
-SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) |
-DART_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, '..', '..', '..')) |
+from shutil import copyfile |
# Dartium DEPS file from the DEPS file checked into the dart-lang/sdk integration |
# branch. |
-DEPS_GIT = ('https://raw.githubusercontent.com/dart-lang/sdk/' |
- 'integration/tools/deps/dartium.deps/DEPS') |
- |
-# Whitelist of files to keep. |
-WHITELIST = [ |
- r'LICENSE(\S+)', |
- r'(\S+)\.idl', |
- r'(\S+)\.json', |
- r'(\S+)\.py', |
- r'(\S+)\.txt', |
- ] |
- |
-# WebKit / WebCore info. |
-CHROME_TRUNK = "https://src.chromium.org" |
-WEBKIT_URL_PATTERN = r'"dartium_webkit_branch": "(\S+)",' |
-WEBKIT_REV_PATTERN = r'"dartium_webkit_revision": "(\d+)",' |
-WEBCORE_SUBPATH = 'Source/core' |
-MODULES_SUBPATH = 'Source/modules' |
-BINDINGS_SUBPATH = 'Source/bindings' |
- |
-LOCAL_WEBKIT_IDL_PATH = os.path.join(DART_PATH, 'third_party', 'WebCore') |
-LOCAL_WEBKIT_README = """\ |
-This directory contains a copy of WebKit/WebCore IDL files. |
-See the attached LICENSE-* files in this directory. |
- |
-Please do not modify the files here. They are periodically copied |
-using the script: $DART_ROOT/sdk/lib/html/scripts/%(script)s |
- |
-The current version corresponds to: |
-URL: %(url)s |
-Current revision: %(revision)s |
-""" |
- |
-# Chrome info. |
-CHROME_URL_PATTERN = r'"dartium_chromium_branch": "(\S+)",' |
-CHROME_REV_PATTERN = r'"dartium_chromium_revision": "(\d+)",' |
-CHROME_IDL_SUBPATH = 'trunk/src/chrome/common/extensions/api' |
-CHROME_COMMENT_EATER_SUBPATH = 'trunk/src/tools/json_comment_eater' |
-CHROME_COMPILER_SUBPATH = 'trunk/src/tools/json_schema_compiler' |
-CHROME_IDL_PARSER_SUBPATH = 'trunk/src/ppapi/generators' |
-CHROME_PLY_SUBPATH = 'trunk/src/third_party/ply' |
-LOCAL_CHROME_IDL_PATH = os.path.join(DART_PATH, 'third_party', 'chrome', 'idl') |
-LOCAL_CHROME_COMMENT_EATER_PATH = os.path.join( |
- DART_PATH, 'third_party', 'chrome', 'tools', 'json_comment_eater') |
-LOCAL_CHROME_COMPILER_PATH = os.path.join(DART_PATH, 'third_party', 'chrome', |
- 'tools', 'json_schema_compiler') |
-LOCAL_CHROME_IDL_PARSER_PATH = os.path.join(DART_PATH, 'third_party', 'chrome', |
- 'ppapi', 'generators') |
-LOCAL_CHROME_PLY_PATH = os.path.join(DART_PATH, 'third_party', 'chrome', |
- 'third_party', 'ply') |
-LOCAL_CHROME_README = """\ |
-This directory contains a copy of Chromium IDL and generation scripts |
-used to generate Dart APIs for Chrome Apps. |
- |
-The original files are from: |
-URL: %(url)s |
-Current revision: %(revision)s |
- |
-Please see the corresponding LICENSE file at |
-%(url)s/trunk/src/LICENSE. |
-""" |
-DEPTH_FILES = 'files' |
-DEPTH_INFINITY = 'infinity' |
- |
-# Regular expressions corresponding to URL/revision patters in the |
-# DEPS file. |
+DEPS_GIT = ('https://raw.githubusercontent.com/dart-lang/sdk/integration/' |
+ 'tools/deps/dartium.deps/DEPS') |
+CHROME_TRUNK = "https://chromium.googlesource.com" |
+WEBKIT_URL_PATTERN = r'"dartium_chromium_commit": "(\S+)",' |
DEPS_PATTERNS = { |
- 'webkit': (CHROME_TRUNK, WEBKIT_URL_PATTERN, WEBKIT_REV_PATTERN), |
-# 'chrome': (CHROME_TRUNK, CHROME_URL_PATTERN, CHROME_REV_PATTERN), |
- } |
- |
-# List of components to update. |
-UPDATE_LIST = [ |
- # (component, remote subpath, local path, local readme file, depth) |
- |
- # WebKit IDL. |
- ('webkit', WEBCORE_SUBPATH, os.path.join(LOCAL_WEBKIT_IDL_PATH, 'core'), |
- LOCAL_WEBKIT_README, DEPTH_INFINITY), |
- ('webkit', MODULES_SUBPATH, os.path.join(LOCAL_WEBKIT_IDL_PATH, 'modules'), |
- LOCAL_WEBKIT_README, DEPTH_INFINITY), |
- ('webkit', BINDINGS_SUBPATH, os.path.join(LOCAL_WEBKIT_IDL_PATH, 'bindings'), |
- LOCAL_WEBKIT_README, DEPTH_INFINITY), |
- |
- # Chrome IDL. |
- ('chrome', CHROME_IDL_SUBPATH, LOCAL_CHROME_IDL_PATH, LOCAL_CHROME_README, |
- DEPTH_INFINITY), |
- # Chrome PPAPI generators. Contains idl_parser.py which is used by the |
- # Chrome IDL compiler. |
- ('chrome', CHROME_IDL_PARSER_SUBPATH, LOCAL_CHROME_IDL_PARSER_PATH, |
- LOCAL_CHROME_README, DEPTH_FILES), |
- # ply files. |
- ('chrome', CHROME_PLY_SUBPATH, LOCAL_CHROME_PLY_PATH, LOCAL_CHROME_README, |
- DEPTH_INFINITY), |
- # Path for json_comment_eater, which is needed by the Chrome IDL compiler. |
- ('chrome', CHROME_COMMENT_EATER_SUBPATH, LOCAL_CHROME_COMMENT_EATER_PATH, |
- LOCAL_CHROME_README, DEPTH_FILES), |
- # Chrome IDL compiler files. |
- ('chrome', CHROME_COMPILER_SUBPATH, LOCAL_CHROME_COMPILER_PATH, |
- LOCAL_CHROME_README, DEPTH_INFINITY), |
- ] |
- |
+ 'webkit': (CHROME_TRUNK, WEBKIT_URL_PATTERN), |
+} |
+ |
+# Dartium/Chromium remote (GIT repository) |
+GIT_REMOTES_CHROMIUM = 'https://chromium.googlesource.com/dart/dartium/src.git' |
+ |
+# location of this file |
+SOURCE_FILE_DIR = 'src/dart/tools/dom/scripts' |
+ |
+WEBKIT_SOURCE = 'src/third_party/WebKit/Source' |
+WEBCORE_SOURCE = 'src/dart/third_party/WebCore' |
+ |
+# Never automatically git add bindings/IDLExtendedAttributes.txt this file has |
+# been modified by Dart but is usually changed by WebKit blink too. |
+IDL_EXTENDED_ATTRIBUTES_FILE = 'IDLExtendedAttributes.txt' |
+ |
+# Don't automatically update, delete or add anything in this directory: |
+# bindings/dart/scripts |
+# The scripts in the above directory is the source for our Dart generators that |
+# is driven from the blink IDL parser AST |
+DART_SDK_GENERATOR_SCRIPTS = 'bindings/dart/scripts' |
+ |
+# sub directories containing IDLs (core and modules) from the base directory |
+# src/third_party/WebKit/Source |
+SUBDIRS = [ |
+ 'bindings', |
+ 'core', |
+ 'modules', |
+] |
+IDL_EXT = '.idl' |
+PY_EXT = '.py' |
+LICENSE_FILE_PREFIX = 'LICENSE' # e.g., LICENSE-APPLE, etc. |
+ |
+# Look in any file in WebCore we copy from WebKit if this comment is in the file |
+# then flag this as a special .py or .idl file that needs to be looked at. |
+DART_CHANGES = ' FIXMEDART: ' |
+ |
+# application options passed in. |
+options = None |
+ |
+warning_messages = [] |
+ |
+# Is --check passed in. |
+def isChecked(): |
+ global options |
+ return options['check'] is not None |
+ |
+# Is --verbose passed in. |
+def isVerbose(): |
+ global options |
+ return options['verbose'] is not None |
def RunCommand(cmd, valid_exits=[0]): |
"""Executes a shell command and return its stdout.""" |
- print ' '.join(cmd) |
+ if isVerbose(): |
+ print ' '.join(cmd) |
pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
output = pipe.communicate() |
if pipe.returncode in valid_exits: |
@@ -132,102 +112,207 @@ def RunCommand(cmd, valid_exits=[0]): |
print 'FAILED. RET_CODE=%d' % pipe.returncode |
sys.exit(pipe.returncode) |
+# returns True if // FIXMEDART: is in the file. |
+def anyDartFixMe(filepath): |
+ if os.path.exists(filepath): |
+ data = open(filepath, 'r').read() |
+ return data.find(DART_CHANGES) != -1 |
+ else: |
+ return False |
+ |
+# Give a base_dir compute the trailing directory after base_dir |
+# returns the subpath from base_dir for the path passed in. |
+def subpath(path, base_dir): |
+ dir_portion = '' |
+ head = path |
+ while True: |
+ head, tail = os.path.split(head) |
+ dir_portion = os.path.join(tail, dir_portion) |
+ if head == base_dir or tail == '': |
+ break; |
+ return dir_portion |
+ |
+# Copy any file in source_dir (WebKit) to destination_dir (dart/third_party/WebCore) |
+# source_dir is the src/third_party/WebKit/Source location (blink) |
+# destination_dir is the src/dart/third_party/WebCore location |
+# returns idls_copied, py_copied, other_copied |
+def copy_files(source_dir, destination_dir): |
+ original_cwd = os.getcwd() |
+ os.chdir(destination_dir) |
+ |
+ idls = 0 # *.idl files copied |
+ pys = 0 # *.py files copied |
+ others = 0 # all other files copied |
+ |
+ for (root, _, files) in os.walk(source_dir, topdown=False): |
+ dir_portion = subpath(root, source_dir) |
+ for f in files: |
+ # Never automatically add any Dart generator scripts (these are the original |
+ # sources in WebCore) from WebKit to WebCore. |
+ if dir_portion != DART_SDK_GENERATOR_SCRIPTS: |
+ if (f.endswith(IDL_EXT) or |
+ f == IDL_EXTENDED_ATTRIBUTES_FILE or |
+ f.endswith(PY_EXT) or |
+ f.startswith(LICENSE_FILE_PREFIX)): |
+ if f.endswith(IDL_EXT): |
+ idls += 1 |
+ elif f.endswith(PY_EXT): |
+ pys += 1 |
+ else: |
+ others += 1 |
+ src_file = os.path.join(root, f) |
+ dst_root = root.replace(WEBKIT_SOURCE, WEBCORE_SOURCE) |
+ dst_file = os.path.join(dst_root, f) |
+ |
+ destination = os.path.dirname(dst_file) |
+ if not os.path.exists(destination): |
+ os.makedirs(destination) |
+ |
+ has_Dart_fix_me = anyDartFixMe(dst_file) |
+ |
+ if not isChecked(): |
+ copyfile(src_file, dst_file) |
+ if isVerbose(): |
+ print('...copying %s' % os.path.split(dst_file)[1]) |
+ if f == IDL_EXTENDED_ATTRIBUTES_FILE: |
+ warning_messages.append(dst_file) |
+ else: |
+ if has_Dart_fix_me: |
+ warning_messages.append(dst_file) |
+ if not (isChecked() or has_Dart_fix_me): |
+ # git add the file |
+ RunCommand(['git', 'add', dst_file]) |
+ |
+ os.chdir(original_cwd) |
+ |
+ return [idls, pys, others] |
+ |
+# Remove any file in webcore_dir that no longer exist in the webkit_dir |
+# webcore_dir src/dart/third_party/WebCore location |
+# webkit_dir src/third_party/WebKit/Source location (blink) |
+# only check if the subdir off from webcore_dir |
+# return list of files deleted |
+def remove_obsolete_webcore_files(webcore_dir, webkit_dir, subdir): |
+ files_to_delete = [] |
+ |
+ original_cwd = os.getcwd() |
+ os.chdir(webcore_dir) |
+ |
+ for (root, _, files) in os.walk(os.path.join(webcore_dir, subdir), topdown=False): |
+ dir_portion = subpath(root, webcore_dir) |
+ for f in files: |
+ # Never automatically deleted any Dart generator scripts (these are the |
+ # original sources in WebCore). |
+ if dir_portion != DART_SDK_GENERATOR_SCRIPTS: |
+ check_file = os.path.join(dir_portion, f) |
+ check_file_full_path = os.path.join(webkit_dir, check_file) |
+ if not os.path.exists(check_file_full_path): |
+ if not isChecked(): |
+ # Remove the file using git |
+ RunCommand(['git', 'rm', check_file]) |
+ files_to_delete.append(check_file) |
+ |
+ os.chdir(original_cwd) |
+ |
+ return files_to_delete |
+ |
+def ParseOptions(): |
+ parser = optparse.OptionParser() |
+ parser.add_option('--verbose', '-v', dest='verbose', action='store_false', |
+ help='Dump all information', default=None) |
+ parser.add_option('--check', '-c', dest='check', action='store_false', |
+ help='Display results without adding, updating or deleting any files', default=None) |
+ args, _ = parser.parse_args() |
+ |
+ argOptions = {} |
+ argOptions['verbose'] = args.verbose |
+ argOptions['check'] = args.check |
+ return argOptions |
+ |
+# Fetch the DEPS file in src/dart/tools/deps/dartium.deps/DEPS from the GIT repro. |
def GetDepsFromGit(): |
req = requests.get(DEPS_GIT) |
return req.text |
-def GetSvnRevision(deps, component): |
- """Returns a tuple with the (dartium webkit repo, latest revision).""" |
- url_base, url_pattern, rev_pattern = DEPS_PATTERNS[component] |
- url = url_base + re.search(url_pattern, deps).group(1) |
- revision = re.search(rev_pattern, deps).group(1) |
- return (url, revision) |
+def ValidateGitRemotes(): |
+ #origin https://chromium.googlesource.com/dart/dartium/src.git (fetch) |
+ remotes_list = RunCommand(['git', 'remote', '--verbose']).split() |
+ if (len(remotes_list) > 2 and |
+ remotes_list[0] == 'origin' and remotes_list[1] == GIT_REMOTES_CHROMIUM): |
+ return True |
+ print 'ERROR: Unable to find dart/dartium/src repository %s' % GIT_REMOTES_CHROMIUM |
+ return False |
-def RefreshFiles(url, revision, remote_path, local_path, depth): |
- """Refreshes refreshes files in the local_path to specific url / |
- revision / remote_path, exporting to depth""" |
+def getCurrentDartiumSHA(): |
cwd = os.getcwd() |
- try: |
- if os.path.exists(local_path): |
- shutil.rmtree(local_path) |
- head, tail = os.path.split(local_path) |
- if not os.path.exists(head): |
- os.makedirs(head) |
- os.chdir(head) |
- RunCommand(['svn', 'export', '--depth', depth, '-r', revision, |
- url + '/' + remote_path, tail]) |
- finally: |
- os.chdir(cwd) |
- |
- |
-def PruneExtraFiles(local_path): |
- """Removes all files that do not match the whitelist.""" |
- pattern = re.compile(reduce(lambda x,y: '%s|%s' % (x,y), |
- map(lambda z: '(%s)' % z, WHITELIST))) |
- for (root, dirs, files) in os.walk(local_path, topdown=False): |
- for f in files: |
- if not pattern.match(f): |
- os.remove(os.path.join(root, f)) |
- for d in dirs: |
- dirpath = os.path.join(root, d) |
- if not os.listdir(dirpath): |
- shutil.rmtree(dirpath) |
- |
- |
-def GenerateReadme(local_path, template, url, revision): |
- readme = template % { |
- 'script': os.path.basename(__file__), |
- 'url': url, |
- 'revision': revision } |
- |
- readme_path = os.path.join(local_path, 'README') |
- out = open(readme_path, 'w') |
- out.write(readme) |
- out.close() |
- |
-ZIP_ARCHIVE = 'version-control-dirs.zip' |
- |
-def SaveVersionControlDir(local_path): |
- if os.path.exists(local_path): |
- RunCommand([ |
- 'sh', '-c', |
- 'find %s -name .svn -or -name .git | zip -r %s -@' % ( |
- os.path.relpath(local_path), ZIP_ARCHIVE) |
- ], [0, 12]) # It is ok if zip has nothing to do (exit code 12). |
- |
-def RestoreVersionControlDir(local_path): |
- archive_path = os.path.join(local_path, ZIP_ARCHIVE) |
- if os.path.exists(archive_path): |
- RunCommand(['unzip', ZIP_ARCHIVE, '-d', '.']) |
- RunCommand(['rm', ZIP_ARCHIVE]) |
+ if cwd.endswith('dart'): |
+ # In src/dart |
+ src_dir, _ = os.path.split(cwd) |
+ elif cwd.endswith('src'): |
+ src_dir = cwd |
+ else: |
+ src_dir = os.path.join(cwd, 'src') |
+ os.chdir(src_dir) |
-def ParseOptions(): |
- parser = optparse.OptionParser() |
- parser.add_option('--webkit-revision', '-w', dest='webkit_revision', |
- help='WebKit IDL revision to install', default=None) |
- parser.add_option('--chrome-revision', '-c', dest='chrome_revision', |
- help='Chrome IDL revision to install', default=None) |
- args, _ = parser.parse_args() |
- update = {} |
- update['webkit'] = args.webkit_revision |
- return update |
+ if ValidateGitRemotes(): |
+ dartium_sha = RunCommand(['git', 'log', '--format=format:%H', '-1']) |
+ else: |
+ dartium_sha = -1 |
+ os.chdir(cwd) |
+ return dartium_sha |
+ |
+# Returns the SHA of the Dartium/Chromiun in the DEPS file. |
+def GetDEPSDartiumGitRevision(deps, component): |
+ """Returns a tuple with the (dartium chromium repo, latest revision).""" |
+ url_base, url_pattern = DEPS_PATTERNS[component] |
+ url = url_base + re.search(url_pattern, deps).group(1) |
+ # Get the SHA for the Chromium/WebKit changes for Dartium. |
+ revision = url[len(url_base):] |
+ return revision |
def main(): |
- update = ParseOptions() |
+ global options |
+ options = ParseOptions() |
+ |
+ current_dir = os.path.dirname(os.path.abspath(__file__)) |
+ if not current_dir.endswith(SOURCE_FILE_DIR): |
+ print 'ERROR: idlsync.py not run in proper directory (%s)\n', current_dir |
+ |
+ base_directory = current_dir[:current_dir.rfind(SOURCE_FILE_DIR)] |
+ |
+ # Validate that the DEPS SHA matches the SHA of the chromium/dartium branch. |
deps = GetDepsFromGit() |
- for (component, remote_path, local_path, readme, depth) in UPDATE_LIST: |
- if component in update.keys(): |
- revision = update[component] |
- url, latest = GetSvnRevision(deps, component) |
- if revision is None: |
- revision = latest |
- SaveVersionControlDir(local_path); |
- RefreshFiles(url, revision, remote_path, local_path, depth) |
- PruneExtraFiles(local_path) |
- GenerateReadme(local_path, readme, url, revision) |
- RestoreVersionControlDir(local_path); |
+ revision = GetDEPSDartiumGitRevision(deps, 'webkit') |
+ dartium_sha = getCurrentDartiumSHA() |
+ if not(revision == dartium_sha): |
+ print "ERROR: Chromium/Dartium SHA in DEPS doesn't match the GIT branch." |
+ return |
+ |
+ start_time = time.time() |
+ for subdir in SUBDIRS: |
+ webkit_dir = os.path.join(base_directory, WEBKIT_SOURCE) |
+ webcore_dir = os.path.join(base_directory, WEBCORE_SOURCE) |
+ |
+ idls_deleted = remove_obsolete_webcore_files(webcore_dir, webkit_dir, subdir) |
+ print "%s files removed in WebCore %s" % (idls_deleted.__len__(), subdir) |
+ if isVerbose(): |
+ for delete_file in idls_deleted: |
+ print " %s" % delete_file |
+ |
+ idls_copied, py_copied, other_copied = copy_files(os.path.join(webkit_dir, subdir), webcore_dir) |
+ print "Copied %s IDLs to %s" % (idls_copied, subdir) |
+ print "Copied %s PYs to %s" % (py_copied, subdir) |
+ print "Copied %s other to %s\n" % (other_copied, subdir) |
+ |
+ end_time = time.time() |
+ |
+ print 'WARNING: File(s) contain FIXMEDART and are NOT "git add " please review:' |
+ for warning in warning_messages: |
+ print ' %s' % warning |
+ |
+ print '\nDone idlsync completed in %s seconds' % round(end_time - start_time, 2) |
if __name__ == '__main__': |
- main() |
+ sys.exit(main()) |