Index: android_webview/tools/webview_licenses.py |
diff --git a/android_webview/tools/webview_licenses.py b/android_webview/tools/webview_licenses.py |
index 63a26e21548a7bb6a409ea1c75c159c1108cd790..11c85d1570e8bd012796a96277e1981adb124f68 100755 |
--- a/android_webview/tools/webview_licenses.py |
+++ b/android_webview/tools/webview_licenses.py |
@@ -18,6 +18,7 @@ aren't in a third-party directory with a README.chromium file. |
import glob |
import imp |
+import multiprocessing |
import optparse |
import os |
import re |
@@ -95,6 +96,43 @@ def GetUnknownIncompatibleDirectories(): |
class ScanResult(object): |
Ok, Warnings, Errors = range(3) |
+# Needs to be a top-level function for multiprocessing |
+def _FindCopyrights(files_to_scan): |
+ args = [os.path.join('android_webview', 'tools', 'find_copyrights.pl')] |
+ p = subprocess.Popen( |
+ args=args, cwd=REPOSITORY_ROOT, |
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE) |
+ lines = p.communicate(files_to_scan)[0].splitlines() |
+ |
+ offending_files = [] |
+ allowed_copyrights = '^(?:\*No copyright\*' \ |
+ '|20[0-9][0-9](?:-20[0-9][0-9])? The Chromium Authors\. ' \ |
+ 'All rights reserved.*)$' |
+ allowed_copyrights_re = re.compile(allowed_copyrights) |
+ for l in lines: |
+ entries = l.split('\t') |
+ if entries[1] == "GENERATED FILE": |
+ continue |
+ copyrights = entries[1].split(' / ') |
+ for c in copyrights: |
+ if c and not allowed_copyrights_re.match(c): |
+ offending_files.append(os.path.normpath(entries[0])) |
+ break |
+ return offending_files |
+ |
+def _ShardString(s, delimiter, shard_len): |
+ result = [] |
+ index = 0 |
+ last_pos = 0 |
+ for m in re.finditer(delimiter, s): |
+ index += 1 |
+ if index % shard_len == 0: |
+ result.append(s[last_pos:m.end()]) |
+ last_pos = m.end() |
+ if not index % shard_len == 0: |
+ result.append(s[last_pos:]) |
+ return result |
+ |
def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files): |
"""Checks that all files which are not in a listed third-party directory, |
and which do not use the standard Chromium license, are whitelisted. |
@@ -147,26 +185,21 @@ def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files): |
# This is not part of open source chromium, but are included on some bots. |
excluded_dirs_list.append('skia/tools/clusterfuzz-data') |
- args = ['android_webview/tools/find_copyrights.pl', |
+ args = [os.path.join('android_webview', 'tools', 'find_files.pl'), |
'.' |
] + excluded_dirs_list |
p = subprocess.Popen(args=args, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE) |
- lines = p.communicate()[0].splitlines() |
- |
- offending_files = [] |
- allowed_copyrights = '^(?:\*No copyright\*' \ |
- '|20[0-9][0-9](?:-20[0-9][0-9])? The Chromium Authors\. ' \ |
- 'All rights reserved.*)$' |
- allowed_copyrights_re = re.compile(allowed_copyrights) |
- for l in lines: |
- entries = l.split('\t') |
- if entries[1] == "GENERATED FILE": |
- continue |
- copyrights = entries[1].split(' / ') |
- for c in copyrights: |
- if c and not allowed_copyrights_re.match(c): |
- offending_files.append(os.path.normpath(entries[0])) |
- break |
+ files_to_scan = p.communicate()[0] |
+ |
+ sharded_files_to_scan = _ShardString(files_to_scan, '\n', 2000) |
+ pool = multiprocessing.Pool() |
+ offending_files_chunks = pool.map_async( |
+ _FindCopyrights, sharded_files_to_scan).get(999999) |
+ pool.close() |
+ pool.join() |
+ # Flatten out the result |
+ offending_files = \ |
+ [item for sublist in offending_files_chunks for item in sublist] |
unknown = set(offending_files) - set(whitelisted_files) |
if unknown: |