Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(187)

Side by Side Diff: presubmit_canned_checks.py

Issue 1181103002: Parallelize pylint PRESUBMIT checks. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: fix test Created 5 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | presubmit_support.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Generic presubmit checks that can be reused by other presubmit checks.""" 5 """Generic presubmit checks that can be reused by other presubmit checks."""
6 6
7 import os as _os 7 import os as _os
8 _HERE = _os.path.dirname(_os.path.abspath(__file__)) 8 _HERE = _os.path.dirname(_os.path.abspath(__file__))
9 9
10 # Justifications for each filter: 10 # Justifications for each filter:
(...skipping 753 matching lines...) Expand 10 before | Expand all | Expand 10 after
764 764
765 input_api.logging.info('Running pylint on %d files', len(files)) 765 input_api.logging.info('Running pylint on %d files', len(files))
766 input_api.logging.debug('Running pylint on: %s', files) 766 input_api.logging.debug('Running pylint on: %s', files)
767 # Copy the system path to the environment so pylint can find the right 767 # Copy the system path to the environment so pylint can find the right
768 # imports. 768 # imports.
769 env = input_api.environ.copy() 769 env = input_api.environ.copy()
770 import sys 770 import sys
771 env['PYTHONPATH'] = input_api.os_path.pathsep.join( 771 env['PYTHONPATH'] = input_api.os_path.pathsep.join(
772 extra_paths_list + sys.path).encode('utf8') 772 extra_paths_list + sys.path).encode('utf8')
773 773
774 def GetPylintCmd(files): 774 def GetPylintCmd(files, extra, parallel):
775 # Windows needs help running python files so we explicitly specify 775 # Windows needs help running python files so we explicitly specify
776 # the interpreter to use. It also has limitations on the size of 776 # the interpreter to use. It also has limitations on the size of
777 # the command-line, so we pass arguments via a pipe. 777 # the command-line, so we pass arguments via a pipe.
778 cmd = [input_api.python_executable,
779 input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
780 '--args-on-stdin']
778 if len(files) == 1: 781 if len(files) == 1:
779 description = files[0] 782 description = files[0]
780 else: 783 else:
781 description = '%s files' % len(files) 784 description = '%s files' % len(files)
782 785
786 if extra:
787 cmd.extend(extra)
788 description += ' using %s' % (extra,)
789 if parallel:
790 cmd.append('--jobs=%s' % input_api.cpu_count)
791 description += ' on %d cores' % input_api.cpu_count
792
783 return input_api.Command( 793 return input_api.Command(
784 name='Pylint (%s)' % description, 794 name='Pylint (%s)' % description,
785 cmd=[input_api.python_executable, 795 cmd=cmd,
786 input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
787 '--args-on-stdin'],
788 kwargs={'env': env, 'stdin': '\n'.join(files + extra_args)}, 796 kwargs={'env': env, 'stdin': '\n'.join(files + extra_args)},
789 message=error_type) 797 message=error_type)
790 798
791 # Always run pylint and pass it all the py files at once. 799 # Always run pylint and pass it all the py files at once.
792 # Passing py files one at time is slower and can produce 800 # Passing py files one at time is slower and can produce
793 # different results. input_api.verbose used to be used 801 # different results. input_api.verbose used to be used
794 # to enable this behaviour but differing behaviour in 802 # to enable this behaviour but differing behaviour in
795 # verbose mode is not desirable. 803 # verbose mode is not desirable.
796 # Leave this unreachable code in here so users can make 804 # Leave this unreachable code in here so users can make
797 # a quick local edit to diagnose pylint issues more 805 # a quick local edit to diagnose pylint issues more
798 # easily. 806 # easily.
799 if True: 807 if True:
800 return [GetPylintCmd(files)] 808 # pylint's cycle detection doesn't work in parallel, so spawn a second,
809 # single-threaded job for just that check.
810 return [
811 GetPylintCmd(files, ["--disable=cyclic-import"], True),
812 GetPylintCmd(files, ["--disable=all", "--enable=cyclic-import"], False)
813 ]
801 else: 814 else:
802 return map(lambda x: GetPylintCmd([x]), files) 815 return map(lambda x: GetPylintCmd([x], extra_args, 1), files)
803 816
804 817
805 def RunPylint(input_api, *args, **kwargs): 818 def RunPylint(input_api, *args, **kwargs):
806 """Legacy presubmit function. 819 """Legacy presubmit function.
807 820
808 For better performance, get all tests and then pass to 821 For better performance, get all tests and then pass to
809 input_api.RunTests. 822 input_api.RunTests.
810 """ 823 """
811 return input_api.RunTests(GetPylint(input_api, *args, **kwargs), False) 824 return input_api.RunTests(GetPylint(input_api, *args, **kwargs), False)
812 825
(...skipping 316 matching lines...) Expand 10 before | Expand all | Expand 10 after
1129 for f in affected_files: 1142 for f in affected_files:
1130 cmd = ['gn', 'format', '--dry-run', f.AbsoluteLocalPath()] 1143 cmd = ['gn', 'format', '--dry-run', f.AbsoluteLocalPath()]
1131 rc = gn.main(cmd) 1144 rc = gn.main(cmd)
1132 if rc == 2: 1145 if rc == 2:
1133 warnings.append(output_api.PresubmitPromptWarning( 1146 warnings.append(output_api.PresubmitPromptWarning(
1134 '%s requires formatting. Please run `gn format --in-place %s`.' % ( 1147 '%s requires formatting. Please run `gn format --in-place %s`.' % (
1135 f.AbsoluteLocalPath(), f.LocalPath()))) 1148 f.AbsoluteLocalPath(), f.LocalPath())))
1136 # It's just a warning, so ignore other types of failures assuming they'll be 1149 # It's just a warning, so ignore other types of failures assuming they'll be
1137 # caught elsewhere. 1150 # caught elsewhere.
1138 return warnings 1151 return warnings
OLDNEW
« no previous file with comments | « no previous file | presubmit_support.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698