OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Checks third-party licenses for the purposes of the Android WebView build. | 6 """Checks third-party licenses for the purposes of the Android WebView build. |
7 | 7 |
8 The Android tree includes a snapshot of Chromium in order to power the system | 8 The Android tree includes a snapshot of Chromium in order to power the system |
9 WebView. This tool checks that all code uses open-source licenses compatible | 9 WebView. This tool checks that all code uses open-source licenses compatible |
10 with Android, and that we meet the requirements of those licenses. It can also | 10 with Android, and that we meet the requirements of those licenses. It can also |
11 be used to generate an Android NOTICE file for the third-party code. | 11 be used to generate an Android NOTICE file for the third-party code. |
12 | 12 |
13 It makes use of src/tools/licenses.py and the README.chromium files on which | 13 It makes use of src/tools/licenses.py and the README.chromium files on which |
14 it depends. It also makes use of a data file, third_party_files_whitelist.txt, | 14 it depends. It also makes use of a data file, third_party_files_whitelist.txt, |
15 which whitelists indicidual files which contain third-party code but which | 15 which whitelists indicidual files which contain third-party code but which |
16 aren't in a third-party directory with a README.chromium file. | 16 aren't in a third-party directory with a README.chromium file. |
17 """ | 17 """ |
18 | 18 |
19 import glob | |
20 import imp | 19 import imp |
21 import json | 20 import json |
22 import multiprocessing | 21 import multiprocessing |
23 import optparse | 22 import optparse |
24 import os | 23 import os |
25 import re | 24 import re |
26 import sys | 25 import sys |
27 import textwrap | 26 import textwrap |
28 | 27 |
29 | 28 |
(...skipping 20 matching lines...) Expand all Loading... |
50 self.os_path = os.path | 49 self.os_path = os.path |
51 self.os_walk = os.walk | 50 self.os_walk = os.walk |
52 self.re = re | 51 self.re = re |
53 self.ReadFile = _ReadFile | 52 self.ReadFile = _ReadFile |
54 self.change = InputApiChange() | 53 self.change = InputApiChange() |
55 | 54 |
56 class InputApiChange(object): | 55 class InputApiChange(object): |
57 def __init__(self): | 56 def __init__(self): |
58 self.RepositoryRoot = lambda: REPOSITORY_ROOT | 57 self.RepositoryRoot = lambda: REPOSITORY_ROOT |
59 | 58 |
60 | |
61 def GetIncompatibleDirectories(): | |
62 """Gets a list of third-party directories which use licenses incompatible | |
63 with Android. This is used by the snapshot tool. | |
64 Returns: | |
65 A list of directories. | |
66 """ | |
67 | |
68 result = [] | |
69 for directory in _FindThirdPartyDirs(): | |
70 if directory in known_issues.KNOWN_ISSUES: | |
71 result.append(directory) | |
72 continue | |
73 try: | |
74 metadata = licenses.ParseDir(directory, REPOSITORY_ROOT, | |
75 require_license_file=False, | |
76 optional_keys=['License Android Compatible']) | |
77 except licenses.LicenseError as e: | |
78 print 'Got LicenseError while scanning ' + directory | |
79 raise | |
80 if metadata.get('License Android Compatible', 'no').upper() == 'YES': | |
81 continue | |
82 license = re.split(' [Ll]icenses?$', metadata['License'])[0] | |
83 if not third_party.LicenseIsCompatibleWithAndroid(InputApi(), license): | |
84 result.append(directory) | |
85 return result | |
86 | |
87 def GetUnknownIncompatibleDirectories(): | |
88 """Gets a list of third-party directories which use licenses incompatible | |
89 with Android which are not present in the known_issues.py file. | |
90 This is used by the AOSP bot. | |
91 Returns: | |
92 A list of directories. | |
93 """ | |
94 incompatible_directories = frozenset(GetIncompatibleDirectories()) | |
95 known_incompatible = [] | |
96 input_api = InputApi() | |
97 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): | |
98 path = copyright_scanner.ForwardSlashesToOsPathSeps(input_api, path) | |
99 for exclude in exclude_list: | |
100 exclude = copyright_scanner.ForwardSlashesToOsPathSeps(input_api, exclude) | |
101 if glob.has_magic(exclude): | |
102 exclude_dirname = os.path.dirname(exclude) | |
103 if glob.has_magic(exclude_dirname): | |
104 print ('Exclude path %s contains an unexpected glob expression,' \ | |
105 ' skipping.' % exclude) | |
106 exclude = exclude_dirname | |
107 known_incompatible.append(os.path.normpath(os.path.join(path, exclude))) | |
108 known_incompatible = frozenset(known_incompatible) | |
109 return incompatible_directories.difference(known_incompatible) | |
110 | |
111 | |
112 class ScanResult(object): | 59 class ScanResult(object): |
113 Ok, Warnings, Errors = range(3) | 60 Ok, Warnings, Errors = range(3) |
114 | 61 |
115 # Needs to be a top-level function for multiprocessing | 62 # Needs to be a top-level function for multiprocessing |
116 def _FindCopyrightViolations(files_to_scan_as_string): | 63 def _FindCopyrightViolations(files_to_scan_as_string): |
117 return copyright_scanner.FindCopyrightViolations( | 64 return copyright_scanner.FindCopyrightViolations( |
118 InputApi(), REPOSITORY_ROOT, files_to_scan_as_string) | 65 InputApi(), REPOSITORY_ROOT, files_to_scan_as_string) |
119 | 66 |
120 def _ShardList(l, shard_len): | 67 def _ShardList(l, shard_len): |
121 return [l[i:i + shard_len] for i in range(0, len(l), shard_len)] | 68 return [l[i:i + shard_len] for i in range(0, len(l), shard_len)] |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
177 Args: | 124 Args: |
178 full_path: The path of the file to read. | 125 full_path: The path of the file to read. |
179 Returns: | 126 Returns: |
180 The contents of the file as a string. | 127 The contents of the file as a string. |
181 """ | 128 """ |
182 | 129 |
183 with open(full_path, mode) as f: | 130 with open(full_path, mode) as f: |
184 return f.read() | 131 return f.read() |
185 | 132 |
186 | 133 |
187 def _ReadLocalFile(path, mode='rb'): | |
188 """Reads a file from disk. | |
189 Args: | |
190 path: The path of the file to read, relative to the root of the repository. | |
191 Returns: | |
192 The contents of the file as a string. | |
193 """ | |
194 | |
195 return _ReadFile(os.path.join(REPOSITORY_ROOT, path), mode) | |
196 | |
197 | |
198 def _FindThirdPartyDirs(): | 134 def _FindThirdPartyDirs(): |
199 """Gets the list of third-party directories. | 135 """Gets the list of third-party directories. |
200 Returns: | 136 Returns: |
201 The list of third-party directories. | 137 The list of third-party directories. |
202 """ | 138 """ |
203 | 139 |
204 # Please don't add here paths that have problems with license files, | 140 # Please don't add here paths that have problems with license files, |
205 # as they will end up included in Android WebView snapshot. | 141 # as they will end up included in Android WebView snapshot. |
206 # Instead, add them into known_issues.py. | 142 # Instead, add them into known_issues.py. |
207 prune_paths = [ | 143 prune_paths = [ |
(...skipping 130 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
338 if generate_licenses_file_list_only: | 274 if generate_licenses_file_list_only: |
339 return [entry['license_file'] for entry in entries] | 275 return [entry['license_file'] for entry in entries] |
340 else: | 276 else: |
341 env = jinja2.Environment( | 277 env = jinja2.Environment( |
342 loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), | 278 loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), |
343 extensions=['jinja2.ext.autoescape']) | 279 extensions=['jinja2.ext.autoescape']) |
344 template = env.get_template('licenses_notice.tmpl') | 280 template = env.get_template('licenses_notice.tmpl') |
345 return template.render({ 'entries': entries }).encode('utf8') | 281 return template.render({ 'entries': entries }).encode('utf8') |
346 | 282 |
347 | 283 |
348 def _ProcessIncompatibleResult(incompatible_directories): | |
349 if incompatible_directories: | |
350 print ("Incompatibly licensed directories found:\n" + | |
351 "\n".join(sorted(incompatible_directories))) | |
352 return ScanResult.Errors | |
353 return ScanResult.Ok | |
354 | |
355 def main(): | 284 def main(): |
356 class FormatterWithNewLines(optparse.IndentedHelpFormatter): | 285 class FormatterWithNewLines(optparse.IndentedHelpFormatter): |
357 def format_description(self, description): | 286 def format_description(self, description): |
358 paras = description.split('\n') | 287 paras = description.split('\n') |
359 formatted_paras = [textwrap.fill(para, self.width) for para in paras] | 288 formatted_paras = [textwrap.fill(para, self.width) for para in paras] |
360 return '\n'.join(formatted_paras) + '\n' | 289 return '\n'.join(formatted_paras) + '\n' |
361 | 290 |
362 parser = optparse.OptionParser(formatter=FormatterWithNewLines(), | 291 parser = optparse.OptionParser(formatter=FormatterWithNewLines(), |
363 usage='%prog [options]') | 292 usage='%prog [options]') |
364 parser.add_option('--json', help='Path to JSON output file') | 293 parser.add_option('--json', help='Path to JSON output file') |
365 parser.description = (__doc__ + | 294 parser.description = (__doc__ + |
366 '\nCommands:\n' | 295 '\nCommands:\n' |
367 ' scan Check licenses.\n' | 296 ' scan Check licenses.\n' |
368 ' notice_deps Generate the list of dependencies for ' | 297 ' notice_deps Generate the list of dependencies for ' |
369 'Android NOTICE file.\n' | 298 'Android NOTICE file.\n' |
370 ' notice [file] Generate Android NOTICE file on ' | 299 ' notice [file] Generate Android NOTICE file on ' |
371 'stdout or into |file|.\n' | 300 'stdout or into |file|.\n' |
372 ' incompatible_directories Scan for incompatibly' | |
373 ' licensed directories.\n' | |
374 ' all_incompatible_directories Scan for incompatibly' | |
375 ' licensed directories (even those in' | |
376 ' known_issues.py).\n' | |
377 ' display_copyrights Display autorship on the files' | 301 ' display_copyrights Display autorship on the files' |
378 ' using names provided via stdin.\n') | 302 ' using names provided via stdin.\n') |
379 (options, args) = parser.parse_args() | 303 (options, args) = parser.parse_args() |
380 if len(args) < 1: | 304 if len(args) < 1: |
381 parser.print_help() | 305 parser.print_help() |
382 return ScanResult.Errors | 306 return ScanResult.Errors |
383 | 307 |
384 if args[0] == 'scan': | 308 if args[0] == 'scan': |
385 scan_result, problem_paths = _Scan() | 309 scan_result, problem_paths = _Scan() |
386 if scan_result == ScanResult.Ok: | 310 if scan_result == ScanResult.Ok: |
387 print 'OK!' | 311 print 'OK!' |
388 if options.json: | 312 if options.json: |
389 with open(options.json, 'w') as f: | 313 with open(options.json, 'w') as f: |
390 json.dump(problem_paths, f) | 314 json.dump(problem_paths, f) |
391 return scan_result | 315 return scan_result |
392 elif args[0] == 'notice_deps': | 316 elif args[0] == 'notice_deps': |
393 # 'set' is used to eliminate duplicate references to the same license file. | 317 # 'set' is used to eliminate duplicate references to the same license file. |
394 print ' '.join( | 318 print ' '.join( |
395 sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True)))) | 319 sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True)))) |
396 return ScanResult.Ok | 320 return ScanResult.Ok |
397 elif args[0] == 'notice': | 321 elif args[0] == 'notice': |
398 notice_file_contents = GenerateNoticeFile() | 322 notice_file_contents = GenerateNoticeFile() |
399 if len(args) == 1: | 323 if len(args) == 1: |
400 print notice_file_contents | 324 print notice_file_contents |
401 else: | 325 else: |
402 with open(args[1], 'w') as output_file: | 326 with open(args[1], 'w') as output_file: |
403 output_file.write(notice_file_contents) | 327 output_file.write(notice_file_contents) |
404 return ScanResult.Ok | 328 return ScanResult.Ok |
405 elif args[0] == 'incompatible_directories': | |
406 return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories()) | |
407 elif args[0] == 'all_incompatible_directories': | |
408 return _ProcessIncompatibleResult(GetIncompatibleDirectories()) | |
409 elif args[0] == 'display_copyrights': | 329 elif args[0] == 'display_copyrights': |
410 files = sys.stdin.read().splitlines() | 330 files = sys.stdin.read().splitlines() |
411 for f, c in \ | 331 for f, c in \ |
412 zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)): | 332 zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)): |
413 print f, '\t', ' / '.join(sorted(c)) | 333 print f, '\t', ' / '.join(sorted(c)) |
414 return ScanResult.Ok | 334 return ScanResult.Ok |
415 parser.print_help() | 335 parser.print_help() |
416 return ScanResult.Errors | 336 return ScanResult.Errors |
417 | 337 |
418 if __name__ == '__main__': | 338 if __name__ == '__main__': |
419 sys.exit(main()) | 339 sys.exit(main()) |
OLD | NEW |