| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Checks third-party licenses for the purposes of the Android WebView build. | 6 """Checks third-party licenses for the purposes of the Android WebView build. |
| 7 | 7 |
| 8 The Android tree includes a snapshot of Chromium in order to power the system | 8 The Android tree includes a snapshot of Chromium in order to power the system |
| 9 WebView. This tool checks that all code uses open-source licenses compatible | 9 WebView. This tool checks that all code uses open-source licenses compatible |
| 10 with Android, and that we meet the requirements of those licenses. It can also | 10 with Android, and that we meet the requirements of those licenses. It can also |
| 11 be used to generate an Android NOTICE file for the third-party code. | 11 be used to generate an Android NOTICE file for the third-party code. |
| 12 | 12 |
| 13 It makes use of src/tools/licenses.py and the README.chromium files on which | 13 It makes use of src/tools/licenses.py and the README.chromium files on which |
| 14 it depends. It also makes use of a data file, third_party_files_whitelist.txt, | 14 it depends. It also makes use of a data file, third_party_files_whitelist.txt, |
| 15 which whitelists indicidual files which contain third-party code but which | 15 which whitelists indicidual files which contain third-party code but which |
| 16 aren't in a third-party directory with a README.chromium file. | 16 aren't in a third-party directory with a README.chromium file. |
| 17 """ | 17 """ |
| 18 | 18 |
| 19 import glob | |
| 20 import imp | 19 import imp |
| 21 import json | 20 import json |
| 22 import multiprocessing | 21 import multiprocessing |
| 23 import optparse | 22 import optparse |
| 24 import os | 23 import os |
| 25 import re | 24 import re |
| 26 import sys | 25 import sys |
| 27 import textwrap | 26 import textwrap |
| 28 | 27 |
| 29 | 28 |
| (...skipping 20 matching lines...) Expand all Loading... |
| 50 self.os_path = os.path | 49 self.os_path = os.path |
| 51 self.os_walk = os.walk | 50 self.os_walk = os.walk |
| 52 self.re = re | 51 self.re = re |
| 53 self.ReadFile = _ReadFile | 52 self.ReadFile = _ReadFile |
| 54 self.change = InputApiChange() | 53 self.change = InputApiChange() |
| 55 | 54 |
| 56 class InputApiChange(object): | 55 class InputApiChange(object): |
| 57 def __init__(self): | 56 def __init__(self): |
| 58 self.RepositoryRoot = lambda: REPOSITORY_ROOT | 57 self.RepositoryRoot = lambda: REPOSITORY_ROOT |
| 59 | 58 |
| 60 | |
| 61 def GetIncompatibleDirectories(): | |
| 62 """Gets a list of third-party directories which use licenses incompatible | |
| 63 with Android. This is used by the snapshot tool. | |
| 64 Returns: | |
| 65 A list of directories. | |
| 66 """ | |
| 67 | |
| 68 result = [] | |
| 69 for directory in _FindThirdPartyDirs(): | |
| 70 if directory in known_issues.KNOWN_ISSUES: | |
| 71 result.append(directory) | |
| 72 continue | |
| 73 try: | |
| 74 metadata = licenses.ParseDir(directory, REPOSITORY_ROOT, | |
| 75 require_license_file=False, | |
| 76 optional_keys=['License Android Compatible']) | |
| 77 except licenses.LicenseError as e: | |
| 78 print 'Got LicenseError while scanning ' + directory | |
| 79 raise | |
| 80 if metadata.get('License Android Compatible', 'no').upper() == 'YES': | |
| 81 continue | |
| 82 license = re.split(' [Ll]icenses?$', metadata['License'])[0] | |
| 83 if not third_party.LicenseIsCompatibleWithAndroid(InputApi(), license): | |
| 84 result.append(directory) | |
| 85 return result | |
| 86 | |
| 87 def GetUnknownIncompatibleDirectories(): | |
| 88 """Gets a list of third-party directories which use licenses incompatible | |
| 89 with Android which are not present in the known_issues.py file. | |
| 90 This is used by the AOSP bot. | |
| 91 Returns: | |
| 92 A list of directories. | |
| 93 """ | |
| 94 incompatible_directories = frozenset(GetIncompatibleDirectories()) | |
| 95 known_incompatible = [] | |
| 96 input_api = InputApi() | |
| 97 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): | |
| 98 path = copyright_scanner.ForwardSlashesToOsPathSeps(input_api, path) | |
| 99 for exclude in exclude_list: | |
| 100 exclude = copyright_scanner.ForwardSlashesToOsPathSeps(input_api, exclude) | |
| 101 if glob.has_magic(exclude): | |
| 102 exclude_dirname = os.path.dirname(exclude) | |
| 103 if glob.has_magic(exclude_dirname): | |
| 104 print ('Exclude path %s contains an unexpected glob expression,' \ | |
| 105 ' skipping.' % exclude) | |
| 106 exclude = exclude_dirname | |
| 107 known_incompatible.append(os.path.normpath(os.path.join(path, exclude))) | |
| 108 known_incompatible = frozenset(known_incompatible) | |
| 109 return incompatible_directories.difference(known_incompatible) | |
| 110 | |
| 111 | |
| 112 class ScanResult(object): | 59 class ScanResult(object): |
| 113 Ok, Warnings, Errors = range(3) | 60 Ok, Warnings, Errors = range(3) |
| 114 | 61 |
| 115 # Needs to be a top-level function for multiprocessing | 62 # Needs to be a top-level function for multiprocessing |
| 116 def _FindCopyrightViolations(files_to_scan_as_string): | 63 def _FindCopyrightViolations(files_to_scan_as_string): |
| 117 return copyright_scanner.FindCopyrightViolations( | 64 return copyright_scanner.FindCopyrightViolations( |
| 118 InputApi(), REPOSITORY_ROOT, files_to_scan_as_string) | 65 InputApi(), REPOSITORY_ROOT, files_to_scan_as_string) |
| 119 | 66 |
| 120 def _ShardList(l, shard_len): | 67 def _ShardList(l, shard_len): |
| 121 return [l[i:i + shard_len] for i in range(0, len(l), shard_len)] | 68 return [l[i:i + shard_len] for i in range(0, len(l), shard_len)] |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 177 Args: | 124 Args: |
| 178 full_path: The path of the file to read. | 125 full_path: The path of the file to read. |
| 179 Returns: | 126 Returns: |
| 180 The contents of the file as a string. | 127 The contents of the file as a string. |
| 181 """ | 128 """ |
| 182 | 129 |
| 183 with open(full_path, mode) as f: | 130 with open(full_path, mode) as f: |
| 184 return f.read() | 131 return f.read() |
| 185 | 132 |
| 186 | 133 |
| 187 def _ReadLocalFile(path, mode='rb'): | |
| 188 """Reads a file from disk. | |
| 189 Args: | |
| 190 path: The path of the file to read, relative to the root of the repository. | |
| 191 Returns: | |
| 192 The contents of the file as a string. | |
| 193 """ | |
| 194 | |
| 195 return _ReadFile(os.path.join(REPOSITORY_ROOT, path), mode) | |
| 196 | |
| 197 | |
| 198 def _FindThirdPartyDirs(): | 134 def _FindThirdPartyDirs(): |
| 199 """Gets the list of third-party directories. | 135 """Gets the list of third-party directories. |
| 200 Returns: | 136 Returns: |
| 201 The list of third-party directories. | 137 The list of third-party directories. |
| 202 """ | 138 """ |
| 203 | 139 |
| 204 # Please don't add here paths that have problems with license files, | 140 # Please don't add here paths that have problems with license files, |
| 205 # as they will end up included in Android WebView snapshot. | 141 # as they will end up included in Android WebView snapshot. |
| 206 # Instead, add them into known_issues.py. | 142 # Instead, add them into known_issues.py. |
| 207 prune_paths = [ | 143 prune_paths = [ |
| (...skipping 130 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 338 if generate_licenses_file_list_only: | 274 if generate_licenses_file_list_only: |
| 339 return [entry['license_file'] for entry in entries] | 275 return [entry['license_file'] for entry in entries] |
| 340 else: | 276 else: |
| 341 env = jinja2.Environment( | 277 env = jinja2.Environment( |
| 342 loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), | 278 loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), |
| 343 extensions=['jinja2.ext.autoescape']) | 279 extensions=['jinja2.ext.autoescape']) |
| 344 template = env.get_template('licenses_notice.tmpl') | 280 template = env.get_template('licenses_notice.tmpl') |
| 345 return template.render({ 'entries': entries }).encode('utf8') | 281 return template.render({ 'entries': entries }).encode('utf8') |
| 346 | 282 |
| 347 | 283 |
| 348 def _ProcessIncompatibleResult(incompatible_directories): | |
| 349 if incompatible_directories: | |
| 350 print ("Incompatibly licensed directories found:\n" + | |
| 351 "\n".join(sorted(incompatible_directories))) | |
| 352 return ScanResult.Errors | |
| 353 return ScanResult.Ok | |
| 354 | |
| 355 def main(): | 284 def main(): |
| 356 class FormatterWithNewLines(optparse.IndentedHelpFormatter): | 285 class FormatterWithNewLines(optparse.IndentedHelpFormatter): |
| 357 def format_description(self, description): | 286 def format_description(self, description): |
| 358 paras = description.split('\n') | 287 paras = description.split('\n') |
| 359 formatted_paras = [textwrap.fill(para, self.width) for para in paras] | 288 formatted_paras = [textwrap.fill(para, self.width) for para in paras] |
| 360 return '\n'.join(formatted_paras) + '\n' | 289 return '\n'.join(formatted_paras) + '\n' |
| 361 | 290 |
| 362 parser = optparse.OptionParser(formatter=FormatterWithNewLines(), | 291 parser = optparse.OptionParser(formatter=FormatterWithNewLines(), |
| 363 usage='%prog [options]') | 292 usage='%prog [options]') |
| 364 parser.add_option('--json', help='Path to JSON output file') | 293 parser.add_option('--json', help='Path to JSON output file') |
| 365 parser.description = (__doc__ + | 294 parser.description = (__doc__ + |
| 366 '\nCommands:\n' | 295 '\nCommands:\n' |
| 367 ' scan Check licenses.\n' | 296 ' scan Check licenses.\n' |
| 368 ' notice_deps Generate the list of dependencies for ' | 297 ' notice_deps Generate the list of dependencies for ' |
| 369 'Android NOTICE file.\n' | 298 'Android NOTICE file.\n' |
| 370 ' notice [file] Generate Android NOTICE file on ' | 299 ' notice [file] Generate Android NOTICE file on ' |
| 371 'stdout or into |file|.\n' | 300 'stdout or into |file|.\n' |
| 372 ' incompatible_directories Scan for incompatibly' | |
| 373 ' licensed directories.\n' | |
| 374 ' all_incompatible_directories Scan for incompatibly' | |
| 375 ' licensed directories (even those in' | |
| 376 ' known_issues.py).\n' | |
| 377 ' display_copyrights Display autorship on the files' | 301 ' display_copyrights Display autorship on the files' |
| 378 ' using names provided via stdin.\n') | 302 ' using names provided via stdin.\n') |
| 379 (options, args) = parser.parse_args() | 303 (options, args) = parser.parse_args() |
| 380 if len(args) < 1: | 304 if len(args) < 1: |
| 381 parser.print_help() | 305 parser.print_help() |
| 382 return ScanResult.Errors | 306 return ScanResult.Errors |
| 383 | 307 |
| 384 if args[0] == 'scan': | 308 if args[0] == 'scan': |
| 385 scan_result, problem_paths = _Scan() | 309 scan_result, problem_paths = _Scan() |
| 386 if scan_result == ScanResult.Ok: | 310 if scan_result == ScanResult.Ok: |
| 387 print 'OK!' | 311 print 'OK!' |
| 388 if options.json: | 312 if options.json: |
| 389 with open(options.json, 'w') as f: | 313 with open(options.json, 'w') as f: |
| 390 json.dump(problem_paths, f) | 314 json.dump(problem_paths, f) |
| 391 return scan_result | 315 return scan_result |
| 392 elif args[0] == 'notice_deps': | 316 elif args[0] == 'notice_deps': |
| 393 # 'set' is used to eliminate duplicate references to the same license file. | 317 # 'set' is used to eliminate duplicate references to the same license file. |
| 394 print ' '.join( | 318 print ' '.join( |
| 395 sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True)))) | 319 sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True)))) |
| 396 return ScanResult.Ok | 320 return ScanResult.Ok |
| 397 elif args[0] == 'notice': | 321 elif args[0] == 'notice': |
| 398 notice_file_contents = GenerateNoticeFile() | 322 notice_file_contents = GenerateNoticeFile() |
| 399 if len(args) == 1: | 323 if len(args) == 1: |
| 400 print notice_file_contents | 324 print notice_file_contents |
| 401 else: | 325 else: |
| 402 with open(args[1], 'w') as output_file: | 326 with open(args[1], 'w') as output_file: |
| 403 output_file.write(notice_file_contents) | 327 output_file.write(notice_file_contents) |
| 404 return ScanResult.Ok | 328 return ScanResult.Ok |
| 405 elif args[0] == 'incompatible_directories': | |
| 406 return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories()) | |
| 407 elif args[0] == 'all_incompatible_directories': | |
| 408 return _ProcessIncompatibleResult(GetIncompatibleDirectories()) | |
| 409 elif args[0] == 'display_copyrights': | 329 elif args[0] == 'display_copyrights': |
| 410 files = sys.stdin.read().splitlines() | 330 files = sys.stdin.read().splitlines() |
| 411 for f, c in \ | 331 for f, c in \ |
| 412 zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)): | 332 zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)): |
| 413 print f, '\t', ' / '.join(sorted(c)) | 333 print f, '\t', ' / '.join(sorted(c)) |
| 414 return ScanResult.Ok | 334 return ScanResult.Ok |
| 415 parser.print_help() | 335 parser.print_help() |
| 416 return ScanResult.Errors | 336 return ScanResult.Errors |
| 417 | 337 |
| 418 if __name__ == '__main__': | 338 if __name__ == '__main__': |
| 419 sys.exit(main()) | 339 sys.exit(main()) |
| OLD | NEW |