| Index: Tools/Scripts/webkitpy/thirdparty/autopep8.py
|
| diff --git a/Tools/Scripts/webkitpy/thirdparty/autopep8.py b/Tools/Scripts/webkitpy/thirdparty/autopep8.py
|
| new file mode 100644
|
| index 0000000000000000000000000000000000000000..c54ebea44a0bd9c46ca98c24a672c6155eea04cf
|
| --- /dev/null
|
| +++ b/Tools/Scripts/webkitpy/thirdparty/autopep8.py
|
| @@ -0,0 +1,3664 @@
|
| +# Copyright (C) 2010-2011 Hideo Hattori
|
| +# Copyright (C) 2011-2013 Hideo Hattori, Steven Myint
|
| +# Copyright (C) 2013-2014 Hideo Hattori, Steven Myint, Bill Wendling
|
| +#
|
| +# Permission is hereby granted, free of charge, to any person obtaining
|
| +# a copy of this software and associated documentation files (the
|
| +# "Software"), to deal in the Software without restriction, including
|
| +# without limitation the rights to use, copy, modify, merge, publish,
|
| +# distribute, sublicense, and/or sell copies of the Software, and to
|
| +# permit persons to whom the Software is furnished to do so, subject to
|
| +# the following conditions:
|
| +#
|
| +# The above copyright notice and this permission notice shall be
|
| +# included in all copies or substantial portions of the Software.
|
| +#
|
| +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
| +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
| +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
| +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
| +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
| +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
| +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| +# SOFTWARE.
|
| +
|
| +"""Automatically formats Python code to conform to the PEP 8 style guide.
|
| +
|
| +Fixes that only need be done once can be added by adding a function of the form
|
| +"fix_<code>(source)" to this module. They should return the fixed source code.
|
| +These fixes are picked up by apply_global_fixes().
|
| +
|
| +Fixes that depend on pep8 should be added as methods to FixPEP8. See the class
|
| +documentation for more information.
|
| +
|
| +"""
|
| +
|
| +from __future__ import absolute_import
|
| +from __future__ import division
|
| +from __future__ import print_function
|
| +from __future__ import unicode_literals
|
| +
|
| +import bisect
|
| +import codecs
|
| +import collections
|
| +import copy
|
| +import difflib
|
| +import fnmatch
|
| +import inspect
|
| +import io
|
| +import itertools
|
| +import keyword
|
| +import locale
|
| +import os
|
| +import re
|
| +import signal
|
| +import sys
|
| +import token
|
| +import tokenize
|
| +
|
| +import pep8
|
| +
|
| +
|
| +try:
|
| + unicode
|
| +except NameError:
|
| + unicode = str
|
| +
|
| +
|
| +__version__ = '1.0.3'
|
| +
|
| +
|
| +CR = '\r'
|
| +LF = '\n'
|
| +CRLF = '\r\n'
|
| +
|
| +
|
| +PYTHON_SHEBANG_REGEX = re.compile(r'^#!.*\bpython[23]?\b\s*$')
|
| +
|
| +
|
| +# For generating line shortening candidates.
|
| +SHORTEN_OPERATOR_GROUPS = frozenset([
|
| + frozenset([',']),
|
| + frozenset(['%']),
|
| + frozenset([',', '(', '[', '{']),
|
| + frozenset(['%', '(', '[', '{']),
|
| + frozenset([',', '(', '[', '{', '%', '+', '-', '*', '/', '//']),
|
| + frozenset(['%', '+', '-', '*', '/', '//']),
|
| +])
|
| +
|
| +
|
| +DEFAULT_IGNORE = 'E24'
|
| +DEFAULT_INDENT_SIZE = 4
|
| +
|
| +
|
| +# W602 is handled separately due to the need to avoid "with_traceback".
|
| +CODE_TO_2TO3 = {
|
| + 'E721': ['idioms'],
|
| + 'W601': ['has_key'],
|
| + 'W603': ['ne'],
|
| + 'W604': ['repr'],
|
| + 'W690': ['apply',
|
| + 'except',
|
| + 'exitfunc',
|
| + 'import',
|
| + 'numliterals',
|
| + 'operator',
|
| + 'paren',
|
| + 'reduce',
|
| + 'renames',
|
| + 'standarderror',
|
| + 'sys_exc',
|
| + 'throw',
|
| + 'tuple_params',
|
| + 'xreadlines']}
|
| +
|
| +
|
| +def open_with_encoding(filename, encoding=None, mode='r'):
|
| + """Return opened file with a specific encoding."""
|
| + if not encoding:
|
| + encoding = detect_encoding(filename)
|
| +
|
| + return io.open(filename, mode=mode, encoding=encoding,
|
| + newline='') # Preserve line endings
|
| +
|
| +
|
| +def detect_encoding(filename):
|
| + """Return file encoding."""
|
| + try:
|
| + with open(filename, 'rb') as input_file:
|
| + from lib2to3.pgen2 import tokenize as lib2to3_tokenize
|
| + encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0]
|
| +
|
| + # Check for correctness of encoding
|
| + with open_with_encoding(filename, encoding) as test_file:
|
| + test_file.read()
|
| +
|
| + return encoding
|
| + except (LookupError, SyntaxError, UnicodeDecodeError):
|
| + return 'latin-1'
|
| +
|
| +
|
| +def readlines_from_file(filename):
|
| + """Return contents of file."""
|
| + with open_with_encoding(filename) as input_file:
|
| + return input_file.readlines()
|
| +
|
| +
|
| +def extended_blank_lines(logical_line,
|
| + blank_lines,
|
| + indent_level,
|
| + previous_logical):
|
| + """Check for missing blank lines after class declaration."""
|
| + if previous_logical.startswith('class '):
|
| + if (
|
| + logical_line.startswith(('def ', 'class ', '@')) or
|
| + pep8.DOCSTRING_REGEX.match(logical_line)
|
| + ):
|
| + if indent_level and not blank_lines:
|
| + yield (0, 'E309 expected 1 blank line after class declaration')
|
| + elif previous_logical.startswith('def '):
|
| + if blank_lines and pep8.DOCSTRING_REGEX.match(logical_line):
|
| + yield (0, 'E303 too many blank lines ({0})'.format(blank_lines))
|
| + elif pep8.DOCSTRING_REGEX.match(previous_logical):
|
| + # Missing blank line between class docstring and method declaration.
|
| + if (
|
| + indent_level and
|
| + not blank_lines and
|
| + logical_line.startswith(('def ')) and
|
| + '(self' in logical_line
|
| + ):
|
| + yield (0, 'E301 expected 1 blank line, found 0')
|
| +pep8.register_check(extended_blank_lines)
|
| +
|
| +
|
| +def continued_indentation(logical_line, tokens, indent_level, indent_char,
|
| + noqa):
|
| + """Override pep8's function to provide indentation information."""
|
| + first_row = tokens[0][2][0]
|
| + nrows = 1 + tokens[-1][2][0] - first_row
|
| + if noqa or nrows == 1:
|
| + return
|
| +
|
| + # indent_next tells us whether the next block is indented. Assuming
|
| + # that it is indented by 4 spaces, then we should not allow 4-space
|
| + # indents on the final continuation line. In turn, some other
|
| + # indents are allowed to have an extra 4 spaces.
|
| + indent_next = logical_line.endswith(':')
|
| +
|
| + row = depth = 0
|
| + valid_hangs = (
|
| + (DEFAULT_INDENT_SIZE,)
|
| + if indent_char != '\t' else (DEFAULT_INDENT_SIZE,
|
| + 2 * DEFAULT_INDENT_SIZE)
|
| + )
|
| +
|
| + # Remember how many brackets were opened on each line.
|
| + parens = [0] * nrows
|
| +
|
| + # Relative indents of physical lines.
|
| + rel_indent = [0] * nrows
|
| +
|
| + # For each depth, collect a list of opening rows.
|
| + open_rows = [[0]]
|
| + # For each depth, memorize the hanging indentation.
|
| + hangs = [None]
|
| +
|
| + # Visual indents.
|
| + indent_chances = {}
|
| + last_indent = tokens[0][2]
|
| + indent = [last_indent[1]]
|
| +
|
| + last_token_multiline = None
|
| + line = None
|
| + last_line = ''
|
| + last_line_begins_with_multiline = False
|
| + for token_type, text, start, end, line in tokens:
|
| +
|
| + newline = row < start[0] - first_row
|
| + if newline:
|
| + row = start[0] - first_row
|
| + newline = (not last_token_multiline and
|
| + token_type not in (tokenize.NL, tokenize.NEWLINE))
|
| + last_line_begins_with_multiline = last_token_multiline
|
| +
|
| + if newline:
|
| + # This is the beginning of a continuation line.
|
| + last_indent = start
|
| +
|
| + # Record the initial indent.
|
| + rel_indent[row] = pep8.expand_indent(line) - indent_level
|
| +
|
| + # Identify closing bracket.
|
| + close_bracket = (token_type == tokenize.OP and text in ']})')
|
| +
|
| + # Is the indent relative to an opening bracket line?
|
| + for open_row in reversed(open_rows[depth]):
|
| + hang = rel_indent[row] - rel_indent[open_row]
|
| + hanging_indent = hang in valid_hangs
|
| + if hanging_indent:
|
| + break
|
| + if hangs[depth]:
|
| + hanging_indent = (hang == hangs[depth])
|
| +
|
| + visual_indent = (not close_bracket and hang > 0 and
|
| + indent_chances.get(start[1]))
|
| +
|
| + if close_bracket and indent[depth]:
|
| + # Closing bracket for visual indent.
|
| + if start[1] != indent[depth]:
|
| + yield (start, 'E124 {0}'.format(indent[depth]))
|
| + elif close_bracket and not hang:
|
| + pass
|
| + elif indent[depth] and start[1] < indent[depth]:
|
| + # Visual indent is broken.
|
| + yield (start, 'E128 {0}'.format(indent[depth]))
|
| + elif (hanging_indent or
|
| + (indent_next and
|
| + rel_indent[row] == 2 * DEFAULT_INDENT_SIZE)):
|
| + # Hanging indent is verified.
|
| + if close_bracket:
|
| + yield (start, 'E123 {0}'.format(indent_level +
|
| + rel_indent[open_row]))
|
| + hangs[depth] = hang
|
| + elif visual_indent is True:
|
| + # Visual indent is verified.
|
| + indent[depth] = start[1]
|
| + elif visual_indent in (text, unicode):
|
| + # Ignore token lined up with matching one from a previous line.
|
| + pass
|
| + else:
|
| + one_indented = (indent_level + rel_indent[open_row] +
|
| + DEFAULT_INDENT_SIZE)
|
| + # Indent is broken.
|
| + if hang <= 0:
|
| + error = ('E122', one_indented)
|
| + elif indent[depth]:
|
| + error = ('E127', indent[depth])
|
| + elif hang > DEFAULT_INDENT_SIZE:
|
| + error = ('E126', one_indented)
|
| + else:
|
| + hangs[depth] = hang
|
| + error = ('E121', one_indented)
|
| +
|
| + yield (start, '{0} {1}'.format(*error))
|
| +
|
| + # Look for visual indenting.
|
| + if (parens[row] and token_type not in (tokenize.NL, tokenize.COMMENT)
|
| + and not indent[depth]):
|
| + indent[depth] = start[1]
|
| + indent_chances[start[1]] = True
|
| + # Deal with implicit string concatenation.
|
| + elif (token_type in (tokenize.STRING, tokenize.COMMENT) or
|
| + text in ('u', 'ur', 'b', 'br')):
|
| + indent_chances[start[1]] = unicode
|
| + # Special case for the "if" statement because len("if (") is equal to
|
| + # 4.
|
| + elif not indent_chances and not row and not depth and text == 'if':
|
| + indent_chances[end[1] + 1] = True
|
| + elif text == ':' and line[end[1]:].isspace():
|
| + open_rows[depth].append(row)
|
| +
|
| + # Keep track of bracket depth.
|
| + if token_type == tokenize.OP:
|
| + if text in '([{':
|
| + depth += 1
|
| + indent.append(0)
|
| + hangs.append(None)
|
| + if len(open_rows) == depth:
|
| + open_rows.append([])
|
| + open_rows[depth].append(row)
|
| + parens[row] += 1
|
| + elif text in ')]}' and depth > 0:
|
| + # Parent indents should not be more than this one.
|
| + prev_indent = indent.pop() or last_indent[1]
|
| + hangs.pop()
|
| + for d in range(depth):
|
| + if indent[d] > prev_indent:
|
| + indent[d] = 0
|
| + for ind in list(indent_chances):
|
| + if ind >= prev_indent:
|
| + del indent_chances[ind]
|
| + del open_rows[depth + 1:]
|
| + depth -= 1
|
| + if depth:
|
| + indent_chances[indent[depth]] = True
|
| + for idx in range(row, -1, -1):
|
| + if parens[idx]:
|
| + parens[idx] -= 1
|
| + break
|
| + assert len(indent) == depth + 1
|
| + if (
|
| + start[1] not in indent_chances and
|
| + # This is for purposes of speeding up E121 (GitHub #90).
|
| + not last_line.rstrip().endswith(',')
|
| + ):
|
| + # Allow to line up tokens.
|
| + indent_chances[start[1]] = text
|
| +
|
| + last_token_multiline = (start[0] != end[0])
|
| + if last_token_multiline:
|
| + rel_indent[end[0] - first_row] = rel_indent[row]
|
| +
|
| + last_line = line
|
| +
|
| + if (
|
| + indent_next and
|
| + not last_line_begins_with_multiline and
|
| + pep8.expand_indent(line) == indent_level + DEFAULT_INDENT_SIZE
|
| + ):
|
| + pos = (start[0], indent[0] + 4)
|
| + yield (pos, 'E125 {0}'.format(indent_level +
|
| + 2 * DEFAULT_INDENT_SIZE))
|
| +del pep8._checks['logical_line'][pep8.continued_indentation]
|
| +pep8.register_check(continued_indentation)
|
| +
|
| +
|
| +class FixPEP8(object):
|
| +
|
| + """Fix invalid code.
|
| +
|
| + Fixer methods are prefixed "fix_". The _fix_source() method looks for these
|
| + automatically.
|
| +
|
| + The fixer method can take either one or two arguments (in addition to
|
| + self). The first argument is "result", which is the error information from
|
| + pep8. The second argument, "logical", is required only for logical-line
|
| + fixes.
|
| +
|
| + The fixer method can return the list of modified lines or None. An empty
|
| + list would mean that no changes were made. None would mean that only the
|
| + line reported in the pep8 error was modified. Note that the modified line
|
| + numbers that are returned are indexed at 1. This typically would correspond
|
| + with the line number reported in the pep8 error information.
|
| +
|
| + [fixed method list]
|
| + - e121,e122,e123,e124,e125,e126,e127,e128,e129
|
| + - e201,e202,e203
|
| + - e211
|
| + - e221,e222,e223,e224,e225
|
| + - e231
|
| + - e251
|
| + - e261,e262
|
| + - e271,e272,e273,e274
|
| + - e301,e302,e303
|
| + - e401
|
| + - e502
|
| + - e701,e702
|
| + - e711
|
| + - w291
|
| +
|
| + """
|
| +
|
| + def __init__(self, filename,
|
| + options,
|
| + contents=None,
|
| + long_line_ignore_cache=None):
|
| + self.filename = filename
|
| + if contents is None:
|
| + self.source = readlines_from_file(filename)
|
| + else:
|
| + sio = io.StringIO(contents)
|
| + self.source = sio.readlines()
|
| + self.options = options
|
| + self.indent_word = _get_indentword(''.join(self.source))
|
| +
|
| + self.long_line_ignore_cache = (
|
| + set() if long_line_ignore_cache is None
|
| + else long_line_ignore_cache)
|
| +
|
| + # Many fixers are the same even though pep8 categorizes them
|
| + # differently.
|
| + self.fix_e115 = self.fix_e112
|
| + self.fix_e116 = self.fix_e113
|
| + self.fix_e121 = self._fix_reindent
|
| + self.fix_e122 = self._fix_reindent
|
| + self.fix_e123 = self._fix_reindent
|
| + self.fix_e124 = self._fix_reindent
|
| + self.fix_e126 = self._fix_reindent
|
| + self.fix_e127 = self._fix_reindent
|
| + self.fix_e128 = self._fix_reindent
|
| + self.fix_e129 = self._fix_reindent
|
| + self.fix_e202 = self.fix_e201
|
| + self.fix_e203 = self.fix_e201
|
| + self.fix_e211 = self.fix_e201
|
| + self.fix_e221 = self.fix_e271
|
| + self.fix_e222 = self.fix_e271
|
| + self.fix_e223 = self.fix_e271
|
| + self.fix_e226 = self.fix_e225
|
| + self.fix_e227 = self.fix_e225
|
| + self.fix_e228 = self.fix_e225
|
| + self.fix_e241 = self.fix_e271
|
| + self.fix_e242 = self.fix_e224
|
| + self.fix_e261 = self.fix_e262
|
| + self.fix_e272 = self.fix_e271
|
| + self.fix_e273 = self.fix_e271
|
| + self.fix_e274 = self.fix_e271
|
| + self.fix_e309 = self.fix_e301
|
| + self.fix_e501 = (
|
| + self.fix_long_line_logically if
|
| + options and (options.aggressive >= 2 or options.experimental) else
|
| + self.fix_long_line_physically)
|
| + self.fix_e703 = self.fix_e702
|
| +
|
| + self._ws_comma_done = False
|
| +
|
| + def _fix_source(self, results):
|
| + try:
|
| + (logical_start, logical_end) = _find_logical(self.source)
|
| + logical_support = True
|
| + except (SyntaxError, tokenize.TokenError): # pragma: no cover
|
| + logical_support = False
|
| +
|
| + completed_lines = set()
|
| + for result in sorted(results, key=_priority_key):
|
| + if result['line'] in completed_lines:
|
| + continue
|
| +
|
| + fixed_methodname = 'fix_' + result['id'].lower()
|
| + if hasattr(self, fixed_methodname):
|
| + fix = getattr(self, fixed_methodname)
|
| +
|
| + line_index = result['line'] - 1
|
| + original_line = self.source[line_index]
|
| +
|
| + is_logical_fix = len(inspect.getargspec(fix).args) > 2
|
| + if is_logical_fix:
|
| + logical = None
|
| + if logical_support:
|
| + logical = _get_logical(self.source,
|
| + result,
|
| + logical_start,
|
| + logical_end)
|
| + if logical and set(range(
|
| + logical[0][0] + 1,
|
| + logical[1][0] + 1)).intersection(
|
| + completed_lines):
|
| + continue
|
| +
|
| + modified_lines = fix(result, logical)
|
| + else:
|
| + modified_lines = fix(result)
|
| +
|
| + if modified_lines is None:
|
| + # Force logical fixes to report what they modified.
|
| + assert not is_logical_fix
|
| +
|
| + if self.source[line_index] == original_line:
|
| + modified_lines = []
|
| +
|
| + if modified_lines:
|
| + completed_lines.update(modified_lines)
|
| + elif modified_lines == []: # Empty list means no fix
|
| + if self.options.verbose >= 2:
|
| + print(
|
| + '---> Not fixing {f} on line {l}'.format(
|
| + f=result['id'], l=result['line']),
|
| + file=sys.stderr)
|
| + else: # We assume one-line fix when None.
|
| + completed_lines.add(result['line'])
|
| + else:
|
| + if self.options.verbose >= 3:
|
| + print(
|
| + "---> '{0}' is not defined.".format(fixed_methodname),
|
| + file=sys.stderr)
|
| +
|
| + info = result['info'].strip()
|
| + print('---> {0}:{1}:{2}:{3}'.format(self.filename,
|
| + result['line'],
|
| + result['column'],
|
| + info),
|
| + file=sys.stderr)
|
| +
|
| + def fix(self):
|
| + """Return a version of the source code with PEP 8 violations fixed."""
|
| + pep8_options = {
|
| + 'ignore': self.options.ignore,
|
| + 'select': self.options.select,
|
| + 'max_line_length': self.options.max_line_length,
|
| + }
|
| + results = _execute_pep8(pep8_options, self.source)
|
| +
|
| + if self.options.verbose:
|
| + progress = {}
|
| + for r in results:
|
| + if r['id'] not in progress:
|
| + progress[r['id']] = set()
|
| + progress[r['id']].add(r['line'])
|
| + print('---> {n} issue(s) to fix {progress}'.format(
|
| + n=len(results), progress=progress), file=sys.stderr)
|
| +
|
| + if self.options.line_range:
|
| + start, end = self.options.line_range
|
| + results = [r for r in results
|
| + if start <= r['line'] <= end]
|
| +
|
| + self._fix_source(filter_results(source=''.join(self.source),
|
| + results=results,
|
| + aggressive=self.options.aggressive))
|
| +
|
| + if self.options.line_range:
|
| + # If number of lines has changed then change line_range.
|
| + count = sum(sline.count('\n')
|
| + for sline in self.source[start - 1:end])
|
| + self.options.line_range[1] = start + count - 1
|
| +
|
| + return ''.join(self.source)
|
| +
|
| + def _fix_reindent(self, result):
|
| + """Fix a badly indented line.
|
| +
|
| + This is done by adding or removing from its initial indent only.
|
| +
|
| + """
|
| + num_indent_spaces = int(result['info'].split()[1])
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + self.source[line_index] = ' ' * num_indent_spaces + target.lstrip()
|
| +
|
| + def fix_e112(self, result):
|
| + """Fix under-indented comments."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + if not target.lstrip().startswith('#'):
|
| + # Don't screw with invalid syntax.
|
| + return []
|
| +
|
| + self.source[line_index] = self.indent_word + target
|
| +
|
| + def fix_e113(self, result):
|
| + """Fix over-indented comments."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + indent = _get_indentation(target)
|
| + stripped = target.lstrip()
|
| +
|
| + if not stripped.startswith('#'):
|
| + # Don't screw with invalid syntax.
|
| + return []
|
| +
|
| + self.source[line_index] = indent[1:] + stripped
|
| +
|
| + def fix_e125(self, result):
|
| + """Fix indentation undistinguish from the next logical line."""
|
| + num_indent_spaces = int(result['info'].split()[1])
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + spaces_to_add = num_indent_spaces - len(_get_indentation(target))
|
| + indent = len(_get_indentation(target))
|
| + modified_lines = []
|
| +
|
| + while len(_get_indentation(self.source[line_index])) >= indent:
|
| + self.source[line_index] = (' ' * spaces_to_add +
|
| + self.source[line_index])
|
| + modified_lines.append(1 + line_index) # Line indexed at 1.
|
| + line_index -= 1
|
| +
|
| + return modified_lines
|
| +
|
| + def fix_e201(self, result):
|
| + """Remove extraneous whitespace."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + offset = result['column'] - 1
|
| +
|
| + if is_probably_part_of_multiline(target):
|
| + return []
|
| +
|
| + fixed = fix_whitespace(target,
|
| + offset=offset,
|
| + replacement='')
|
| +
|
| + self.source[line_index] = fixed
|
| +
|
| + def fix_e224(self, result):
|
| + """Remove extraneous whitespace around operator."""
|
| + target = self.source[result['line'] - 1]
|
| + offset = result['column'] - 1
|
| + fixed = target[:offset] + target[offset:].replace('\t', ' ')
|
| + self.source[result['line'] - 1] = fixed
|
| +
|
| + def fix_e225(self, result):
|
| + """Fix missing whitespace around operator."""
|
| + target = self.source[result['line'] - 1]
|
| + offset = result['column'] - 1
|
| + fixed = target[:offset] + ' ' + target[offset:]
|
| +
|
| + # Only proceed if non-whitespace characters match.
|
| + # And make sure we don't break the indentation.
|
| + if (
|
| + fixed.replace(' ', '') == target.replace(' ', '') and
|
| + _get_indentation(fixed) == _get_indentation(target)
|
| + ):
|
| + self.source[result['line'] - 1] = fixed
|
| + else:
|
| + return []
|
| +
|
| + def fix_e231(self, result):
|
| + """Add missing whitespace."""
|
| + # Optimize for comma case. This will fix all commas in the full source
|
| + # code in one pass. Don't do this more than once. If it fails the first
|
| + # time, there is no point in trying again.
|
| + if ',' in result['info'] and not self._ws_comma_done:
|
| + self._ws_comma_done = True
|
| + original = ''.join(self.source)
|
| + new = refactor(original, ['ws_comma'])
|
| + if original.strip() != new.strip():
|
| + self.source = [new]
|
| + return range(1, 1 + len(original))
|
| +
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + offset = result['column']
|
| + fixed = target[:offset] + ' ' + target[offset:]
|
| + self.source[line_index] = fixed
|
| +
|
| + def fix_e251(self, result):
|
| + """Remove whitespace around parameter '=' sign."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + # This is necessary since pep8 sometimes reports columns that goes
|
| + # past the end of the physical line. This happens in cases like,
|
| + # foo(bar\n=None)
|
| + c = min(result['column'] - 1,
|
| + len(target) - 1)
|
| +
|
| + if target[c].strip():
|
| + fixed = target
|
| + else:
|
| + fixed = target[:c].rstrip() + target[c:].lstrip()
|
| +
|
| + # There could be an escaped newline
|
| + #
|
| + # def foo(a=\
|
| + # 1)
|
| + if fixed.endswith(('=\\\n', '=\\\r\n', '=\\\r')):
|
| + self.source[line_index] = fixed.rstrip('\n\r \t\\')
|
| + self.source[line_index + 1] = self.source[line_index + 1].lstrip()
|
| + return [line_index + 1, line_index + 2] # Line indexed at 1
|
| +
|
| + self.source[result['line'] - 1] = fixed
|
| +
|
| + def fix_e262(self, result):
|
| + """Fix spacing after comment hash."""
|
| + target = self.source[result['line'] - 1]
|
| + offset = result['column']
|
| +
|
| + code = target[:offset].rstrip(' \t#')
|
| + comment = target[offset:].lstrip(' \t#')
|
| +
|
| + fixed = code + (' # ' + comment if comment.strip() else '\n')
|
| +
|
| + self.source[result['line'] - 1] = fixed
|
| +
|
| + def fix_e271(self, result):
|
| + """Fix extraneous whitespace around keywords."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + offset = result['column'] - 1
|
| +
|
| + if is_probably_part_of_multiline(target):
|
| + return []
|
| +
|
| + fixed = fix_whitespace(target,
|
| + offset=offset,
|
| + replacement=' ')
|
| +
|
| + if fixed == target:
|
| + return []
|
| + else:
|
| + self.source[line_index] = fixed
|
| +
|
| + def fix_e301(self, result):
|
| + """Add missing blank line."""
|
| + cr = '\n'
|
| + self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]
|
| +
|
| + def fix_e302(self, result):
|
| + """Add missing 2 blank lines."""
|
| + add_linenum = 2 - int(result['info'].split()[-1])
|
| + cr = '\n' * add_linenum
|
| + self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]
|
| +
|
| + def fix_e303(self, result):
|
| + """Remove extra blank lines."""
|
| + delete_linenum = int(result['info'].split('(')[1].split(')')[0]) - 2
|
| + delete_linenum = max(1, delete_linenum)
|
| +
|
| + # We need to count because pep8 reports an offset line number if there
|
| + # are comments.
|
| + cnt = 0
|
| + line = result['line'] - 2
|
| + modified_lines = []
|
| + while cnt < delete_linenum and line >= 0:
|
| + if not self.source[line].strip():
|
| + self.source[line] = ''
|
| + modified_lines.append(1 + line) # Line indexed at 1
|
| + cnt += 1
|
| + line -= 1
|
| +
|
| + return modified_lines
|
| +
|
| + def fix_e304(self, result):
|
| + """Remove blank line following function decorator."""
|
| + line = result['line'] - 2
|
| + if not self.source[line].strip():
|
| + self.source[line] = ''
|
| +
|
| + def fix_e401(self, result):
|
| + """Put imports on separate lines."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + offset = result['column'] - 1
|
| +
|
| + if not target.lstrip().startswith('import'):
|
| + return []
|
| +
|
| + indentation = re.split(pattern=r'\bimport\b',
|
| + string=target, maxsplit=1)[0]
|
| + fixed = (target[:offset].rstrip('\t ,') + '\n' +
|
| + indentation + 'import ' + target[offset:].lstrip('\t ,'))
|
| + self.source[line_index] = fixed
|
| +
|
| + def fix_long_line_logically(self, result, logical):
|
| + """Try to make lines fit within --max-line-length characters."""
|
| + if (
|
| + not logical or
|
| + len(logical[2]) == 1 or
|
| + self.source[result['line'] - 1].lstrip().startswith('#')
|
| + ):
|
| + return self.fix_long_line_physically(result)
|
| +
|
| + start_line_index = logical[0][0]
|
| + end_line_index = logical[1][0]
|
| + logical_lines = logical[2]
|
| +
|
| + previous_line = get_item(self.source, start_line_index - 1, default='')
|
| + next_line = get_item(self.source, end_line_index + 1, default='')
|
| +
|
| + single_line = join_logical_line(''.join(logical_lines))
|
| +
|
| + try:
|
| + fixed = self.fix_long_line(
|
| + target=single_line,
|
| + previous_line=previous_line,
|
| + next_line=next_line,
|
| + original=''.join(logical_lines))
|
| + except (SyntaxError, tokenize.TokenError):
|
| + return self.fix_long_line_physically(result)
|
| +
|
| + if fixed:
|
| + for line_index in range(start_line_index, end_line_index + 1):
|
| + self.source[line_index] = ''
|
| + self.source[start_line_index] = fixed
|
| + return range(start_line_index + 1, end_line_index + 1)
|
| + else:
|
| + return []
|
| +
|
| + def fix_long_line_physically(self, result):
|
| + """Try to make lines fit within --max-line-length characters."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + previous_line = get_item(self.source, line_index - 1, default='')
|
| + next_line = get_item(self.source, line_index + 1, default='')
|
| +
|
| + try:
|
| + fixed = self.fix_long_line(
|
| + target=target,
|
| + previous_line=previous_line,
|
| + next_line=next_line,
|
| + original=target)
|
| + except (SyntaxError, tokenize.TokenError):
|
| + return []
|
| +
|
| + if fixed:
|
| + self.source[line_index] = fixed
|
| + return [line_index + 1]
|
| + else:
|
| + return []
|
| +
|
| + def fix_long_line(self, target, previous_line,
|
| + next_line, original):
|
| + cache_entry = (target, previous_line, next_line)
|
| + if cache_entry in self.long_line_ignore_cache:
|
| + return []
|
| +
|
| + if target.lstrip().startswith('#'):
|
| + # Wrap commented lines.
|
| + return shorten_comment(
|
| + line=target,
|
| + max_line_length=self.options.max_line_length,
|
| + last_comment=not next_line.lstrip().startswith('#'))
|
| +
|
| + fixed = get_fixed_long_line(
|
| + target=target,
|
| + previous_line=previous_line,
|
| + original=original,
|
| + indent_word=self.indent_word,
|
| + max_line_length=self.options.max_line_length,
|
| + aggressive=self.options.aggressive,
|
| + experimental=self.options.experimental,
|
| + verbose=self.options.verbose)
|
| + if fixed and not code_almost_equal(original, fixed):
|
| + return fixed
|
| + else:
|
| + self.long_line_ignore_cache.add(cache_entry)
|
| + return None
|
| +
|
| + def fix_e502(self, result):
|
| + """Remove extraneous escape of newline."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + self.source[line_index] = target.rstrip('\n\r \t\\') + '\n'
|
| +
|
| + def fix_e701(self, result):
|
| + """Put colon-separated compound statement on separate lines."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + c = result['column']
|
| +
|
| + fixed_source = (target[:c] + '\n' +
|
| + _get_indentation(target) + self.indent_word +
|
| + target[c:].lstrip('\n\r \t\\'))
|
| + self.source[result['line'] - 1] = fixed_source
|
| + return [result['line'], result['line'] + 1]
|
| +
|
| + def fix_e702(self, result, logical):
|
| + """Put semicolon-separated compound statement on separate lines."""
|
| + if not logical:
|
| + return [] # pragma: no cover
|
| + logical_lines = logical[2]
|
| +
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + if target.rstrip().endswith('\\'):
|
| + # Normalize '1; \\\n2' into '1; 2'.
|
| + self.source[line_index] = target.rstrip('\n \r\t\\')
|
| + self.source[line_index + 1] = self.source[line_index + 1].lstrip()
|
| + return [line_index + 1, line_index + 2]
|
| +
|
| + if target.rstrip().endswith(';'):
|
| + self.source[line_index] = target.rstrip('\n \r\t;') + '\n'
|
| + return [line_index + 1]
|
| +
|
| + offset = result['column'] - 1
|
| + first = target[:offset].rstrip(';').rstrip()
|
| + second = (_get_indentation(logical_lines[0]) +
|
| + target[offset:].lstrip(';').lstrip())
|
| +
|
| + self.source[line_index] = first + '\n' + second
|
| + return [line_index + 1]
|
| +
|
| + def fix_e711(self, result):
|
| + """Fix comparison with None."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + offset = result['column'] - 1
|
| +
|
| + right_offset = offset + 2
|
| + if right_offset >= len(target):
|
| + return []
|
| +
|
| + left = target[:offset].rstrip()
|
| + center = target[offset:right_offset]
|
| + right = target[right_offset:].lstrip()
|
| +
|
| + if not right.startswith('None'):
|
| + return []
|
| +
|
| + if center.strip() == '==':
|
| + new_center = 'is'
|
| + elif center.strip() == '!=':
|
| + new_center = 'is not'
|
| + else:
|
| + return []
|
| +
|
| + self.source[line_index] = ' '.join([left, new_center, right])
|
| +
|
| + def fix_e712(self, result):
|
| + """Fix comparison with boolean."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| + offset = result['column'] - 1
|
| +
|
| + # Handle very easy "not" special cases.
|
| + if re.match(r'^\s*if \w+ == False:$', target):
|
| + self.source[line_index] = re.sub(r'if (\w+) == False:',
|
| + r'if not \1:', target, count=1)
|
| + elif re.match(r'^\s*if \w+ != True:$', target):
|
| + self.source[line_index] = re.sub(r'if (\w+) != True:',
|
| + r'if not \1:', target, count=1)
|
| + else:
|
| + right_offset = offset + 2
|
| + if right_offset >= len(target):
|
| + return []
|
| +
|
| + left = target[:offset].rstrip()
|
| + center = target[offset:right_offset]
|
| + right = target[right_offset:].lstrip()
|
| +
|
| + # Handle simple cases only.
|
| + new_right = None
|
| + if center.strip() == '==':
|
| + if re.match(r'\bTrue\b', right):
|
| + new_right = re.sub(r'\bTrue\b *', '', right, count=1)
|
| + elif center.strip() == '!=':
|
| + if re.match(r'\bFalse\b', right):
|
| + new_right = re.sub(r'\bFalse\b *', '', right, count=1)
|
| +
|
| + if new_right is None:
|
| + return []
|
| +
|
| + if new_right[0].isalnum():
|
| + new_right = ' ' + new_right
|
| +
|
| + self.source[line_index] = left + new_right
|
| +
|
| + def fix_e713(self, result):
|
| + """Fix non-membership check."""
|
| + line_index = result['line'] - 1
|
| + target = self.source[line_index]
|
| +
|
| + # Handle very easy case only.
|
| + if re.match(r'^\s*if not \w+ in \w+:$', target):
|
| + self.source[line_index] = re.sub(r'if not (\w+) in (\w+):',
|
| + r'if \1 not in \2:',
|
| + target,
|
| + count=1)
|
| +
|
| + def fix_w291(self, result):
|
| + """Remove trailing whitespace."""
|
| + fixed_line = self.source[result['line'] - 1].rstrip()
|
| + self.source[result['line'] - 1] = fixed_line + '\n'
|
| +
|
| +
|
| +def get_fixed_long_line(target, previous_line, original,
|
| + indent_word=' ', max_line_length=79,
|
| + aggressive=False, experimental=False, verbose=False):
|
| + """Break up long line and return result.
|
| +
|
| + Do this by generating multiple reformatted candidates and then
|
| + ranking the candidates to heuristically select the best option.
|
| +
|
| + """
|
| + indent = _get_indentation(target)
|
| + source = target[len(indent):]
|
| + assert source.lstrip() == source
|
| +
|
| + # Check for partial multiline.
|
| + tokens = list(generate_tokens(source))
|
| +
|
| + candidates = shorten_line(
|
| + tokens, source, indent,
|
| + indent_word,
|
| + max_line_length,
|
| + aggressive=aggressive,
|
| + experimental=experimental,
|
| + previous_line=previous_line)
|
| +
|
| + # Also sort alphabetically as a tie breaker (for determinism).
|
| + candidates = sorted(
|
| + sorted(set(candidates).union([target, original])),
|
| + key=lambda x: line_shortening_rank(x,
|
| + indent_word,
|
| + max_line_length,
|
| + experimental))
|
| +
|
| + if verbose >= 4:
|
| + print(('-' * 79 + '\n').join([''] + candidates + ['']),
|
| + file=codecs.getwriter('utf-8')(sys.stderr.buffer
|
| + if hasattr(sys.stderr,
|
| + 'buffer')
|
| + else sys.stderr))
|
| +
|
| + if candidates:
|
| + return candidates[0]
|
| +
|
| +
|
| +def join_logical_line(logical_line):
|
| + """Return single line based on logical line input."""
|
| + indentation = _get_indentation(logical_line)
|
| +
|
| + return indentation + untokenize_without_newlines(
|
| + generate_tokens(logical_line.lstrip())) + '\n'
|
| +
|
| +
|
| +def untokenize_without_newlines(tokens):
|
| + """Return source code based on tokens."""
|
| + text = ''
|
| + last_row = 0
|
| + last_column = -1
|
| +
|
| + for t in tokens:
|
| + token_string = t[1]
|
| + (start_row, start_column) = t[2]
|
| + (end_row, end_column) = t[3]
|
| +
|
| + if start_row > last_row:
|
| + last_column = 0
|
| + if (
|
| + (start_column > last_column or token_string == '\n') and
|
| + not text.endswith(' ')
|
| + ):
|
| + text += ' '
|
| +
|
| + if token_string != '\n':
|
| + text += token_string
|
| +
|
| + last_row = end_row
|
| + last_column = end_column
|
| +
|
| + return text
|
| +
|
| +
|
| +def _find_logical(source_lines):
|
| + # Make a variable which is the index of all the starts of lines.
|
| + logical_start = []
|
| + logical_end = []
|
| + last_newline = True
|
| + parens = 0
|
| + for t in generate_tokens(''.join(source_lines)):
|
| + if t[0] in [tokenize.COMMENT, tokenize.DEDENT,
|
| + tokenize.INDENT, tokenize.NL,
|
| + tokenize.ENDMARKER]:
|
| + continue
|
| + if not parens and t[0] in [tokenize.NEWLINE, tokenize.SEMI]:
|
| + last_newline = True
|
| + logical_end.append((t[3][0] - 1, t[2][1]))
|
| + continue
|
| + if last_newline and not parens:
|
| + logical_start.append((t[2][0] - 1, t[2][1]))
|
| + last_newline = False
|
| + if t[0] == tokenize.OP:
|
| + if t[1] in '([{':
|
| + parens += 1
|
| + elif t[1] in '}])':
|
| + parens -= 1
|
| + return (logical_start, logical_end)
|
| +
|
| +
|
| +def _get_logical(source_lines, result, logical_start, logical_end):
|
| + """Return the logical line corresponding to the result.
|
| +
|
| + Assumes input is already E702-clean.
|
| +
|
| + """
|
| + row = result['line'] - 1
|
| + col = result['column'] - 1
|
| + ls = None
|
| + le = None
|
| + for i in range(0, len(logical_start), 1):
|
| + assert logical_end
|
| + x = logical_end[i]
|
| + if x[0] > row or (x[0] == row and x[1] > col):
|
| + le = x
|
| + ls = logical_start[i]
|
| + break
|
| + if ls is None:
|
| + return None
|
| + original = source_lines[ls[0]:le[0] + 1]
|
| + return ls, le, original
|
| +
|
| +
|
| +def get_item(items, index, default=None):
|
| + if 0 <= index < len(items):
|
| + return items[index]
|
| + else:
|
| + return default
|
| +
|
| +
|
| +def reindent(source, indent_size):
|
| + """Reindent all lines."""
|
| + reindenter = Reindenter(source)
|
| + return reindenter.run(indent_size)
|
| +
|
| +
|
| +def code_almost_equal(a, b):
|
| + """Return True if code is similar.
|
| +
|
| + Ignore whitespace when comparing specific line.
|
| +
|
| + """
|
| + split_a = split_and_strip_non_empty_lines(a)
|
| + split_b = split_and_strip_non_empty_lines(b)
|
| +
|
| + if len(split_a) != len(split_b):
|
| + return False
|
| +
|
| + for index in range(len(split_a)):
|
| + if ''.join(split_a[index].split()) != ''.join(split_b[index].split()):
|
| + return False
|
| +
|
| + return True
|
| +
|
| +
|
| +def split_and_strip_non_empty_lines(text):
|
| + """Return lines split by newline.
|
| +
|
| + Ignore empty lines.
|
| +
|
| + """
|
| + return [line.strip() for line in text.splitlines() if line.strip()]
|
| +
|
| +
|
| +def fix_e265(source, aggressive=False): # pylint: disable=unused-argument
|
| + """Format block comments."""
|
| + if '#' not in source:
|
| + # Optimization.
|
| + return source
|
| +
|
| + ignored_line_numbers = multiline_string_lines(
|
| + source,
|
| + include_docstrings=True) | set(commented_out_code_lines(source))
|
| +
|
| + fixed_lines = []
|
| + sio = io.StringIO(source)
|
| + for (line_number, line) in enumerate(sio.readlines(), start=1):
|
| + if (
|
| + line.lstrip().startswith('#') and
|
| + line_number not in ignored_line_numbers
|
| + ):
|
| + indentation = _get_indentation(line)
|
| + line = line.lstrip()
|
| +
|
| + # Normalize beginning if not a shebang.
|
| + if len(line) > 1:
|
| + if (
|
| + # Leave multiple spaces like '# ' alone.
|
| + (line.count('#') > 1 or line[1].isalnum())
|
| + # Leave stylistic outlined blocks alone.
|
| + and not line.rstrip().endswith('#')
|
| + ):
|
| + line = '# ' + line.lstrip('# \t')
|
| +
|
| + fixed_lines.append(indentation + line)
|
| + else:
|
| + fixed_lines.append(line)
|
| +
|
| + return ''.join(fixed_lines)
|
| +
|
| +
|
| +def refactor(source, fixer_names, ignore=None):
|
| + """Return refactored code using lib2to3.
|
| +
|
| + Skip if ignore string is produced in the refactored code.
|
| +
|
| + """
|
| + from lib2to3 import pgen2
|
| + try:
|
| + new_text = refactor_with_2to3(source,
|
| + fixer_names=fixer_names)
|
| + except (pgen2.parse.ParseError,
|
| + SyntaxError,
|
| + UnicodeDecodeError,
|
| + UnicodeEncodeError):
|
| + return source
|
| +
|
| + if ignore:
|
| + if ignore in new_text and ignore not in source:
|
| + return source
|
| +
|
| + return new_text
|
| +
|
| +
|
| +def code_to_2to3(select, ignore):
|
| + fixes = set()
|
| + for code, fix in CODE_TO_2TO3.items():
|
| + if code_match(code, select=select, ignore=ignore):
|
| + fixes |= set(fix)
|
| + return fixes
|
| +
|
| +
|
| +def fix_2to3(source, aggressive=True, select=None, ignore=None):
|
| + """Fix various deprecated code (via lib2to3)."""
|
| + if not aggressive:
|
| + return source
|
| +
|
| + select = select or []
|
| + ignore = ignore or []
|
| +
|
| + return refactor(source,
|
| + code_to_2to3(select=select,
|
| + ignore=ignore))
|
| +
|
| +
|
| +def fix_w602(source, aggressive=True):
|
| + """Fix deprecated form of raising exception."""
|
| + if not aggressive:
|
| + return source
|
| +
|
| + return refactor(source, ['raise'],
|
| + ignore='with_traceback')
|
| +
|
| +
|
| +def find_newline(source):
|
| + """Return type of newline used in source.
|
| +
|
| + Input is a list of lines.
|
| +
|
| + """
|
| + assert not isinstance(source, unicode)
|
| +
|
| + counter = collections.defaultdict(int)
|
| + for line in source:
|
| + if line.endswith(CRLF):
|
| + counter[CRLF] += 1
|
| + elif line.endswith(CR):
|
| + counter[CR] += 1
|
| + elif line.endswith(LF):
|
| + counter[LF] += 1
|
| +
|
| + return (sorted(counter, key=counter.get, reverse=True) or [LF])[0]
|
| +
|
| +
|
| +def _get_indentword(source):
|
| + """Return indentation type."""
|
| + indent_word = ' ' # Default in case source has no indentation
|
| + try:
|
| + for t in generate_tokens(source):
|
| + if t[0] == token.INDENT:
|
| + indent_word = t[1]
|
| + break
|
| + except (SyntaxError, tokenize.TokenError):
|
| + pass
|
| + return indent_word
|
| +
|
| +
|
| +def _get_indentation(line):
|
| + """Return leading whitespace."""
|
| + if line.strip():
|
| + non_whitespace_index = len(line) - len(line.lstrip())
|
| + return line[:non_whitespace_index]
|
| + else:
|
| + return ''
|
| +
|
| +
|
| +def get_diff_text(old, new, filename):
|
| + """Return text of unified diff between old and new."""
|
| + newline = '\n'
|
| + diff = difflib.unified_diff(
|
| + old, new,
|
| + 'original/' + filename,
|
| + 'fixed/' + filename,
|
| + lineterm=newline)
|
| +
|
| + text = ''
|
| + for line in diff:
|
| + text += line
|
| +
|
| + # Work around missing newline (http://bugs.python.org/issue2142).
|
| + if text and not line.endswith(newline):
|
| + text += newline + r'\ No newline at end of file' + newline
|
| +
|
| + return text
|
| +
|
| +
|
| +def _priority_key(pep8_result):
|
| + """Key for sorting PEP8 results.
|
| +
|
| + Global fixes should be done first. This is important for things like
|
| + indentation.
|
| +
|
| + """
|
| + priority = [
|
| + # Fix multiline colon-based before semicolon based.
|
| + 'e701',
|
| + # Break multiline statements early.
|
| + 'e702',
|
| + # Things that make lines longer.
|
| + 'e225', 'e231',
|
| + # Remove extraneous whitespace before breaking lines.
|
| + 'e201',
|
| + # Shorten whitespace in comment before resorting to wrapping.
|
| + 'e262'
|
| + ]
|
| + middle_index = 10000
|
| + lowest_priority = [
|
| + # We need to shorten lines last since the logical fixer can get in a
|
| + # loop, which causes us to exit early.
|
| + 'e501'
|
| + ]
|
| + key = pep8_result['id'].lower()
|
| + try:
|
| + return priority.index(key)
|
| + except ValueError:
|
| + try:
|
| + return middle_index + lowest_priority.index(key) + 1
|
| + except ValueError:
|
| + return middle_index
|
| +
|
| +
|
| +def shorten_line(tokens, source, indentation, indent_word, max_line_length,
|
| + aggressive=False, experimental=False, previous_line=''):
|
| + """Separate line at OPERATOR.
|
| +
|
| + Multiple candidates will be yielded.
|
| +
|
| + """
|
| + for candidate in _shorten_line(tokens=tokens,
|
| + source=source,
|
| + indentation=indentation,
|
| + indent_word=indent_word,
|
| + aggressive=aggressive,
|
| + previous_line=previous_line):
|
| + yield candidate
|
| +
|
| + if aggressive:
|
| + for key_token_strings in SHORTEN_OPERATOR_GROUPS:
|
| + shortened = _shorten_line_at_tokens(
|
| + tokens=tokens,
|
| + source=source,
|
| + indentation=indentation,
|
| + indent_word=indent_word,
|
| + key_token_strings=key_token_strings,
|
| + aggressive=aggressive)
|
| +
|
| + if shortened is not None and shortened != source:
|
| + yield shortened
|
| +
|
| + if experimental:
|
| + for shortened in _shorten_line_at_tokens_new(
|
| + tokens=tokens,
|
| + source=source,
|
| + indentation=indentation,
|
| + max_line_length=max_line_length):
|
| +
|
| + yield shortened
|
| +
|
| +
|
| +def _shorten_line(tokens, source, indentation, indent_word,
|
| + aggressive=False, previous_line=''):
|
| + """Separate line at OPERATOR.
|
| +
|
| + The input is expected to be free of newlines except for inside multiline
|
| + strings and at the end.
|
| +
|
| + Multiple candidates will be yielded.
|
| +
|
| + """
|
| + for (token_type,
|
| + token_string,
|
| + start_offset,
|
| + end_offset) in token_offsets(tokens):
|
| +
|
| + if (
|
| + token_type == tokenize.COMMENT and
|
| + not is_probably_part_of_multiline(previous_line) and
|
| + not is_probably_part_of_multiline(source) and
|
| + not source[start_offset + 1:].strip().lower().startswith(
|
| + ('noqa', 'pragma:', 'pylint:'))
|
| + ):
|
| + # Move inline comments to previous line.
|
| + first = source[:start_offset]
|
| + second = source[start_offset:]
|
| + yield (indentation + second.strip() + '\n' +
|
| + indentation + first.strip() + '\n')
|
| + elif token_type == token.OP and token_string != '=':
|
| + # Don't break on '=' after keyword as this violates PEP 8.
|
| +
|
| + assert token_type != token.INDENT
|
| +
|
| + first = source[:end_offset]
|
| +
|
| + second_indent = indentation
|
| + if first.rstrip().endswith('('):
|
| + second_indent += indent_word
|
| + elif '(' in first:
|
| + second_indent += ' ' * (1 + first.find('('))
|
| + else:
|
| + second_indent += indent_word
|
| +
|
| + second = (second_indent + source[end_offset:].lstrip())
|
| + if (
|
| + not second.strip() or
|
| + second.lstrip().startswith('#')
|
| + ):
|
| + continue
|
| +
|
| + # Do not begin a line with a comma
|
| + if second.lstrip().startswith(','):
|
| + continue
|
| + # Do end a line with a dot
|
| + if first.rstrip().endswith('.'):
|
| + continue
|
| + if token_string in '+-*/':
|
| + fixed = first + ' \\' + '\n' + second
|
| + else:
|
| + fixed = first + '\n' + second
|
| +
|
| + # Only fix if syntax is okay.
|
| + if check_syntax(normalize_multiline(fixed)
|
| + if aggressive else fixed):
|
| + yield indentation + fixed
|
| +
|
| +
|
| +# A convenient way to handle tokens.
|
| +Token = collections.namedtuple('Token', ['token_type', 'token_string',
|
| + 'spos', 'epos', 'line'])
|
| +
|
| +
|
| +class ReformattedLines(object):
|
| +
|
| + """The reflowed lines of atoms.
|
| +
|
| + Each part of the line is represented as an "atom." They can be moved
|
| + around when need be to get the optimal formatting.
|
| +
|
| + """
|
| +
|
| + ###########################################################################
|
| + # Private Classes
|
| +
|
| + class _Indent(object):
|
| +
|
| + """Represent an indentation in the atom stream."""
|
| +
|
| + def __init__(self, indent_amt):
|
| + self._indent_amt = indent_amt
|
| +
|
| + def emit(self):
|
| + return ' ' * self._indent_amt
|
| +
|
| + @property
|
| + def size(self):
|
| + return self._indent_amt
|
| +
|
| + class _Space(object):
|
| +
|
| + """Represent a space in the atom stream."""
|
| +
|
| + def emit(self):
|
| + return ' '
|
| +
|
| + @property
|
| + def size(self):
|
| + return 1
|
| +
|
| + class _LineBreak(object):
|
| +
|
| + """Represent a line break in the atom stream."""
|
| +
|
| + def emit(self):
|
| + return '\n'
|
| +
|
| + @property
|
| + def size(self):
|
| + return 0
|
| +
|
| + def __init__(self, max_line_length):
|
| + self._max_line_length = max_line_length
|
| + self._lines = []
|
| + self._bracket_depth = 0
|
| + self._prev_item = None
|
| + self._prev_prev_item = None
|
| +
|
| + def __repr__(self):
|
| + return self.emit()
|
| +
|
| + ###########################################################################
|
| + # Public Methods
|
| +
|
| + def add(self, obj, indent_amt, break_after_open_bracket):
|
| + if isinstance(obj, Atom):
|
| + self._add_item(obj, indent_amt)
|
| + return
|
| +
|
| + self._add_container(obj, indent_amt, break_after_open_bracket)
|
| +
|
| + def add_comment(self, item):
|
| + num_spaces = 2
|
| + if len(self._lines) > 1:
|
| + if isinstance(self._lines[-1], self._Space):
|
| + num_spaces -= 1
|
| + if len(self._lines) > 2:
|
| + if isinstance(self._lines[-2], self._Space):
|
| + num_spaces -= 1
|
| +
|
| + while num_spaces > 0:
|
| + self._lines.append(self._Space())
|
| + num_spaces -= 1
|
| + self._lines.append(item)
|
| +
|
| + def add_indent(self, indent_amt):
|
| + self._lines.append(self._Indent(indent_amt))
|
| +
|
| + def add_line_break(self, indent):
|
| + self._lines.append(self._LineBreak())
|
| + self.add_indent(len(indent))
|
| +
|
| + def add_line_break_at(self, index, indent_amt):
|
| + self._lines.insert(index, self._LineBreak())
|
| + self._lines.insert(index + 1, self._Indent(indent_amt))
|
| +
|
| + def add_space_if_needed(self, curr_text, equal=False):
|
| + if (
|
| + not self._lines or isinstance(
|
| + self._lines[-1], (self._LineBreak, self._Indent, self._Space))
|
| + ):
|
| + return
|
| +
|
| + prev_text = unicode(self._prev_item)
|
| + prev_prev_text = (
|
| + unicode(self._prev_prev_item) if self._prev_prev_item else '')
|
| +
|
| + if (
|
| + # The previous item was a keyword or identifier and the current
|
| + # item isn't an operator that doesn't require a space.
|
| + ((self._prev_item.is_keyword or self._prev_item.is_string or
|
| + self._prev_item.is_name or self._prev_item.is_number) and
|
| + (curr_text[0] not in '([{.,:}])' or
|
| + (curr_text[0] == '=' and equal))) or
|
| +
|
| + # Don't place spaces around a '.', unless it's in an 'import'
|
| + # statement.
|
| + ((prev_prev_text != 'from' and prev_text[-1] != '.' and
|
| + curr_text != 'import') and
|
| +
|
| + # Don't place a space before a colon.
|
| + curr_text[0] != ':' and
|
| +
|
| + # Don't split up ending brackets by spaces.
|
| + ((prev_text[-1] in '}])' and curr_text[0] not in '.,}])') or
|
| +
|
| + # Put a space after a colon or comma.
|
| + prev_text[-1] in ':,' or
|
| +
|
| + # Put space around '=' if asked to.
|
| + (equal and prev_text == '=') or
|
| +
|
| + # Put spaces around non-unary arithmetic operators.
|
| + ((self._prev_prev_item and
|
| + (prev_text not in '+-' and
|
| + (self._prev_prev_item.is_name or
|
| + self._prev_prev_item.is_number or
|
| + self._prev_prev_item.is_string)) and
|
| + prev_text in ('+', '-', '%', '*', '/', '//', '**')))))
|
| + ):
|
| + self._lines.append(self._Space())
|
| +
|
| + def previous_item(self):
|
| + """Return the previous non-whitespace item."""
|
| + return self._prev_item
|
| +
|
| + def fits_on_current_line(self, item_extent):
|
| + return self.current_size() + item_extent <= self._max_line_length
|
| +
|
| + def current_size(self):
|
| + """The size of the current line minus the indentation."""
|
| + size = 0
|
| + for item in reversed(self._lines):
|
| + size += item.size
|
| + if isinstance(item, self._LineBreak):
|
| + break
|
| +
|
| + return size
|
| +
|
| + def line_empty(self):
|
| + return (self._lines and
|
| + isinstance(self._lines[-1],
|
| + (self._LineBreak, self._Indent)))
|
| +
|
| + def emit(self):
|
| + string = ''
|
| + for item in self._lines:
|
| + if isinstance(item, self._LineBreak):
|
| + string = string.rstrip()
|
| + string += item.emit()
|
| +
|
| + return string.rstrip() + '\n'
|
| +
|
| + ###########################################################################
|
| + # Private Methods
|
| +
|
| + def _add_item(self, item, indent_amt):
|
| + """Add an item to the line.
|
| +
|
| + Reflow the line to get the best formatting after the item is
|
| + inserted. The bracket depth indicates if the item is being
|
| + inserted inside of a container or not.
|
| +
|
| + """
|
| + if self._prev_item and self._prev_item.is_string and item.is_string:
|
| + # Place consecutive string literals on separate lines.
|
| + self._lines.append(self._LineBreak())
|
| + self._lines.append(self._Indent(indent_amt))
|
| +
|
| + item_text = unicode(item)
|
| + if self._lines and self._bracket_depth:
|
| + # Adding the item into a container.
|
| + self._prevent_default_initializer_splitting(item, indent_amt)
|
| +
|
| + if item_text in '.,)]}':
|
| + self._split_after_delimiter(item, indent_amt)
|
| +
|
| + elif self._lines and not self.line_empty():
|
| + # Adding the item outside of a container.
|
| + if self.fits_on_current_line(len(item_text)):
|
| + self._enforce_space(item)
|
| +
|
| + else:
|
| + # Line break for the new item.
|
| + self._lines.append(self._LineBreak())
|
| + self._lines.append(self._Indent(indent_amt))
|
| +
|
| + self._lines.append(item)
|
| + self._prev_item, self._prev_prev_item = item, self._prev_item
|
| +
|
| + if item_text in '([{':
|
| + self._bracket_depth += 1
|
| +
|
| + elif item_text in '}])':
|
| + self._bracket_depth -= 1
|
| + assert self._bracket_depth >= 0
|
| +
|
| + def _add_container(self, container, indent_amt, break_after_open_bracket):
|
| + actual_indent = indent_amt + 1
|
| +
|
| + if (
|
| + unicode(self._prev_item) != '=' and
|
| + not self.line_empty() and
|
| + not self.fits_on_current_line(
|
| + container.size + self._bracket_depth + 2)
|
| + ):
|
| +
|
| + if unicode(container)[0] == '(' and self._prev_item.is_name:
|
| + # Don't split before the opening bracket of a call.
|
| + break_after_open_bracket = True
|
| + actual_indent = indent_amt + 4
|
| + elif (
|
| + break_after_open_bracket or
|
| + unicode(self._prev_item) not in '([{'
|
| + ):
|
| + # If the container doesn't fit on the current line and the
|
| + # current line isn't empty, place the container on the next
|
| + # line.
|
| + self._lines.append(self._LineBreak())
|
| + self._lines.append(self._Indent(indent_amt))
|
| + break_after_open_bracket = False
|
| + else:
|
| + actual_indent = self.current_size() + 1
|
| + break_after_open_bracket = False
|
| +
|
| + if isinstance(container, (ListComprehension, IfExpression)):
|
| + actual_indent = indent_amt
|
| +
|
| + # Increase the continued indentation only if recursing on a
|
| + # container.
|
| + container.reflow(self, ' ' * actual_indent,
|
| + break_after_open_bracket=break_after_open_bracket)
|
| +
|
| + def _prevent_default_initializer_splitting(self, item, indent_amt):
|
| + """Prevent splitting between a default initializer.
|
| +
|
| + When there is a default initializer, it's best to keep it all on
|
| + the same line. It's nicer and more readable, even if it goes
|
| + over the maximum allowable line length. This goes back along the
|
| + current line to determine if we have a default initializer, and,
|
| + if so, to remove extraneous whitespaces and add a line
|
| + break/indent before it if needed.
|
| +
|
| + """
|
| + if unicode(item) == '=':
|
| + # This is the assignment in the initializer. Just remove spaces for
|
| + # now.
|
| + self._delete_whitespace()
|
| + return
|
| +
|
| + if (not self._prev_item or not self._prev_prev_item or
|
| + unicode(self._prev_item) != '='):
|
| + return
|
| +
|
| + self._delete_whitespace()
|
| + prev_prev_index = self._lines.index(self._prev_prev_item)
|
| +
|
| + if (
|
| + isinstance(self._lines[prev_prev_index - 1], self._Indent) or
|
| + self.fits_on_current_line(item.size + 1)
|
| + ):
|
| + # The default initializer is already the only item on this line.
|
| + # Don't insert a newline here.
|
| + return
|
| +
|
| + # Replace the space with a newline/indent combo.
|
| + if isinstance(self._lines[prev_prev_index - 1], self._Space):
|
| + del self._lines[prev_prev_index - 1]
|
| +
|
| + self.add_line_break_at(self._lines.index(self._prev_prev_item),
|
| + indent_amt)
|
| +
|
| + def _split_after_delimiter(self, item, indent_amt):
|
| + """Split the line only after a delimiter."""
|
| + self._delete_whitespace()
|
| +
|
| + if self.fits_on_current_line(item.size):
|
| + return
|
| +
|
| + last_space = None
|
| + for item in reversed(self._lines):
|
| + if (
|
| + last_space and
|
| + (not isinstance(item, Atom) or not item.is_colon)
|
| + ):
|
| + break
|
| + else:
|
| + last_space = None
|
| + if isinstance(item, self._Space):
|
| + last_space = item
|
| + if isinstance(item, (self._LineBreak, self._Indent)):
|
| + return
|
| +
|
| + if not last_space:
|
| + return
|
| +
|
| + self.add_line_break_at(self._lines.index(last_space), indent_amt)
|
| +
|
| + def _enforce_space(self, item):
|
| + """Enforce a space in certain situations.
|
| +
|
| + There are cases where we will want a space where normally we
|
| + wouldn't put one. This just enforces the addition of a space.
|
| +
|
| + """
|
| + if isinstance(self._lines[-1],
|
| + (self._Space, self._LineBreak, self._Indent)):
|
| + return
|
| +
|
| + if not self._prev_item:
|
| + return
|
| +
|
| + item_text = unicode(item)
|
| + prev_text = unicode(self._prev_item)
|
| +
|
| + # Prefer a space around a '.' in an import statement, and between the
|
| + # 'import' and '('.
|
| + if (
|
| + (item_text == '.' and prev_text == 'from') or
|
| + (item_text == 'import' and prev_text == '.') or
|
| + (item_text == '(' and prev_text == 'import')
|
| + ):
|
| + self._lines.append(self._Space())
|
| +
|
| + def _delete_whitespace(self):
|
| + """Delete all whitespace from the end of the line."""
|
| + while isinstance(self._lines[-1], (self._Space, self._LineBreak,
|
| + self._Indent)):
|
| + del self._lines[-1]
|
| +
|
| +
|
| +class Atom(object):
|
| +
|
| + """The smallest unbreakable unit that can be reflowed."""
|
| +
|
| + def __init__(self, atom):
|
| + self._atom = atom
|
| +
|
| + def __repr__(self):
|
| + return self._atom.token_string
|
| +
|
| + def __len__(self):
|
| + return self.size
|
| +
|
| + def reflow(
|
| + self, reflowed_lines, continued_indent, extent,
|
| + break_after_open_bracket=False,
|
| + is_list_comp_or_if_expr=False,
|
| + next_is_dot=False
|
| + ):
|
| + if self._atom.token_type == tokenize.COMMENT:
|
| + reflowed_lines.add_comment(self)
|
| + return
|
| +
|
| + total_size = extent if extent else self.size
|
| +
|
| + if self._atom.token_string not in ',:([{}])':
|
| + # Some atoms will need an extra 1-sized space token after them.
|
| + total_size += 1
|
| +
|
| + prev_item = reflowed_lines.previous_item()
|
| + if (
|
| + not is_list_comp_or_if_expr and
|
| + not reflowed_lines.fits_on_current_line(total_size) and
|
| + not (next_is_dot and
|
| + reflowed_lines.fits_on_current_line(self.size + 1)) and
|
| + not reflowed_lines.line_empty() and
|
| + not self.is_colon and
|
| + not (prev_item and prev_item.is_name and
|
| + unicode(self) == '(')
|
| + ):
|
| + # Start a new line if there is already something on the line and
|
| + # adding this atom would make it go over the max line length.
|
| + reflowed_lines.add_line_break(continued_indent)
|
| + else:
|
| + reflowed_lines.add_space_if_needed(unicode(self))
|
| +
|
| + reflowed_lines.add(self, len(continued_indent),
|
| + break_after_open_bracket)
|
| +
|
| + def emit(self):
|
| + return self.__repr__()
|
| +
|
| + @property
|
| + def is_keyword(self):
|
| + return keyword.iskeyword(self._atom.token_string)
|
| +
|
| + @property
|
| + def is_string(self):
|
| + return self._atom.token_type == tokenize.STRING
|
| +
|
| + @property
|
| + def is_name(self):
|
| + return self._atom.token_type == tokenize.NAME
|
| +
|
| + @property
|
| + def is_number(self):
|
| + return self._atom.token_type == tokenize.NUMBER
|
| +
|
| + @property
|
| + def is_comma(self):
|
| + return self._atom.token_string == ','
|
| +
|
| + @property
|
| + def is_colon(self):
|
| + return self._atom.token_string == ':'
|
| +
|
| + @property
|
| + def size(self):
|
| + return len(self._atom.token_string)
|
| +
|
| +
|
| +class Container(object):
|
| +
|
| + """Base class for all container types."""
|
| +
|
| + def __init__(self, items):
|
| + self._items = items
|
| +
|
| + def __repr__(self):
|
| + string = ''
|
| + last_was_keyword = False
|
| +
|
| + for item in self._items:
|
| + if item.is_comma:
|
| + string += ', '
|
| + elif item.is_colon:
|
| + string += ': '
|
| + else:
|
| + item_string = unicode(item)
|
| + if (
|
| + string and
|
| + (last_was_keyword or
|
| + (not string.endswith(tuple('([{,.:}]) ')) and
|
| + not item_string.startswith(tuple('([{,.:}])'))))
|
| + ):
|
| + string += ' '
|
| + string += item_string
|
| +
|
| + last_was_keyword = item.is_keyword
|
| + return string
|
| +
|
| + def __iter__(self):
|
| + for element in self._items:
|
| + yield element
|
| +
|
| + def __getitem__(self, idx):
|
| + return self._items[idx]
|
| +
|
| + def reflow(self, reflowed_lines, continued_indent,
|
| + break_after_open_bracket=False):
|
| + last_was_container = False
|
| + for (index, item) in enumerate(self._items):
|
| + next_item = get_item(self._items, index + 1)
|
| +
|
| + if isinstance(item, Atom):
|
| + is_list_comp_or_if_expr = (
|
| + isinstance(self, (ListComprehension, IfExpression)))
|
| + item.reflow(reflowed_lines, continued_indent,
|
| + self._get_extent(index),
|
| + is_list_comp_or_if_expr=is_list_comp_or_if_expr,
|
| + next_is_dot=(next_item and
|
| + unicode(next_item) == '.'))
|
| + if last_was_container and item.is_comma:
|
| + reflowed_lines.add_line_break(continued_indent)
|
| + last_was_container = False
|
| + else: # isinstance(item, Container)
|
| + reflowed_lines.add(item, len(continued_indent),
|
| + break_after_open_bracket)
|
| + last_was_container = not isinstance(item, (ListComprehension,
|
| + IfExpression))
|
| +
|
| + if (
|
| + break_after_open_bracket and index == 0 and
|
| + # Prefer to keep empty containers together instead of
|
| + # separating them.
|
| + unicode(item) == self.open_bracket and
|
| + (not next_item or unicode(next_item) != self.close_bracket) and
|
| + (len(self._items) != 3 or not isinstance(next_item, Atom))
|
| + ):
|
| + reflowed_lines.add_line_break(continued_indent)
|
| + break_after_open_bracket = False
|
| + else:
|
| + next_next_item = get_item(self._items, index + 2)
|
| + if (
|
| + unicode(item) not in ['.', '%', 'in'] and
|
| + next_item and not isinstance(next_item, Container) and
|
| + unicode(next_item) != ':' and
|
| + next_next_item and (not isinstance(next_next_item, Atom) or
|
| + unicode(next_item) == 'not') and
|
| + not reflowed_lines.line_empty() and
|
| + not reflowed_lines.fits_on_current_line(
|
| + self._get_extent(index + 1) + 2)
|
| + ):
|
| + reflowed_lines.add_line_break(continued_indent)
|
| +
|
| + def _get_extent(self, index):
|
| + """The extent of the full element.
|
| +
|
| + E.g., the length of a function call or keyword.
|
| +
|
| + """
|
| + extent = 0
|
| + prev_item = get_item(self._items, index - 1)
|
| + seen_dot = prev_item and unicode(prev_item) == '.'
|
| + while index < len(self._items):
|
| + item = get_item(self._items, index)
|
| + index += 1
|
| +
|
| + if isinstance(item, (ListComprehension, IfExpression)):
|
| + break
|
| +
|
| + if isinstance(item, Container):
|
| + if prev_item and prev_item.is_name:
|
| + if seen_dot:
|
| + extent += 1
|
| + else:
|
| + extent += item.size
|
| +
|
| + prev_item = item
|
| + continue
|
| + elif (unicode(item) not in ['.', '=', ':', 'not'] and
|
| + not item.is_name and not item.is_string):
|
| + break
|
| +
|
| + if unicode(item) == '.':
|
| + seen_dot = True
|
| +
|
| + extent += item.size
|
| + prev_item = item
|
| +
|
| + return extent
|
| +
|
| + @property
|
| + def is_string(self):
|
| + return False
|
| +
|
| + @property
|
| + def size(self):
|
| + return len(self.__repr__())
|
| +
|
| + @property
|
| + def is_keyword(self):
|
| + return False
|
| +
|
| + @property
|
| + def is_name(self):
|
| + return False
|
| +
|
| + @property
|
| + def is_comma(self):
|
| + return False
|
| +
|
| + @property
|
| + def is_colon(self):
|
| + return False
|
| +
|
| + @property
|
| + def open_bracket(self):
|
| + return None
|
| +
|
| + @property
|
| + def close_bracket(self):
|
| + return None
|
| +
|
| +
|
| +class Tuple(Container):
|
| +
|
| + """A high-level representation of a tuple."""
|
| +
|
| + @property
|
| + def open_bracket(self):
|
| + return '('
|
| +
|
| + @property
|
| + def close_bracket(self):
|
| + return ')'
|
| +
|
| +
|
| +class List(Container):
|
| +
|
| + """A high-level representation of a list."""
|
| +
|
| + @property
|
| + def open_bracket(self):
|
| + return '['
|
| +
|
| + @property
|
| + def close_bracket(self):
|
| + return ']'
|
| +
|
| +
|
| +class DictOrSet(Container):
|
| +
|
| + """A high-level representation of a dictionary or set."""
|
| +
|
| + @property
|
| + def open_bracket(self):
|
| + return '{'
|
| +
|
| + @property
|
| + def close_bracket(self):
|
| + return '}'
|
| +
|
| +
|
| +class ListComprehension(Container):
|
| +
|
| + """A high-level representation of a list comprehension."""
|
| +
|
| + @property
|
| + def size(self):
|
| + length = 0
|
| + for item in self._items:
|
| + if isinstance(item, IfExpression):
|
| + break
|
| + length += item.size
|
| + return length
|
| +
|
| +
|
| +class IfExpression(Container):
|
| +
|
| + """A high-level representation of an if-expression."""
|
| +
|
| +
|
| +def _parse_container(tokens, index, for_or_if=None):
|
| + """Parse a high-level container, such as a list, tuple, etc."""
|
| +
|
| + # Store the opening bracket.
|
| + items = [Atom(Token(*tokens[index]))]
|
| + index += 1
|
| +
|
| + num_tokens = len(tokens)
|
| + while index < num_tokens:
|
| + tok = Token(*tokens[index])
|
| +
|
| + if tok.token_string in ',)]}':
|
| + # First check if we're at the end of a list comprehension or
|
| + # if-expression. Don't add the ending token as part of the list
|
| + # comprehension or if-expression, because they aren't part of those
|
| + # constructs.
|
| + if for_or_if == 'for':
|
| + return (ListComprehension(items), index - 1)
|
| +
|
| + elif for_or_if == 'if':
|
| + return (IfExpression(items), index - 1)
|
| +
|
| + # We've reached the end of a container.
|
| + items.append(Atom(tok))
|
| +
|
| + # If not, then we are at the end of a container.
|
| + if tok.token_string == ')':
|
| + # The end of a tuple.
|
| + return (Tuple(items), index)
|
| +
|
| + elif tok.token_string == ']':
|
| + # The end of a list.
|
| + return (List(items), index)
|
| +
|
| + elif tok.token_string == '}':
|
| + # The end of a dictionary or set.
|
| + return (DictOrSet(items), index)
|
| +
|
| + elif tok.token_string in '([{':
|
| + # A sub-container is being defined.
|
| + (container, index) = _parse_container(tokens, index)
|
| + items.append(container)
|
| +
|
| + elif tok.token_string == 'for':
|
| + (container, index) = _parse_container(tokens, index, 'for')
|
| + items.append(container)
|
| +
|
| + elif tok.token_string == 'if':
|
| + (container, index) = _parse_container(tokens, index, 'if')
|
| + items.append(container)
|
| +
|
| + else:
|
| + items.append(Atom(tok))
|
| +
|
| + index += 1
|
| +
|
| + return (None, None)
|
| +
|
| +
|
| +def _parse_tokens(tokens):
|
| + """Parse the tokens.
|
| +
|
| + This converts the tokens into a form where we can manipulate them
|
| + more easily.
|
| +
|
| + """
|
| +
|
| + index = 0
|
| + parsed_tokens = []
|
| +
|
| + num_tokens = len(tokens)
|
| + while index < num_tokens:
|
| + tok = Token(*tokens[index])
|
| +
|
| + assert tok.token_type != token.INDENT
|
| + if tok.token_type == tokenize.NEWLINE:
|
| + # There's only one newline and it's at the end.
|
| + break
|
| +
|
| + if tok.token_string in '([{':
|
| + (container, index) = _parse_container(tokens, index)
|
| + if not container:
|
| + return None
|
| + parsed_tokens.append(container)
|
| + else:
|
| + parsed_tokens.append(Atom(tok))
|
| +
|
| + index += 1
|
| +
|
| + return parsed_tokens
|
| +
|
| +
|
| +def _reflow_lines(parsed_tokens, indentation, max_line_length,
|
| + start_on_prefix_line):
|
| + """Reflow the lines so that it looks nice."""
|
| +
|
| + if unicode(parsed_tokens[0]) == 'def':
|
| + # A function definition gets indented a bit more.
|
| + continued_indent = indentation + ' ' * 2 * DEFAULT_INDENT_SIZE
|
| + else:
|
| + continued_indent = indentation + ' ' * DEFAULT_INDENT_SIZE
|
| +
|
| + break_after_open_bracket = not start_on_prefix_line
|
| +
|
| + lines = ReformattedLines(max_line_length)
|
| + lines.add_indent(len(indentation.lstrip('\r\n')))
|
| +
|
| + if not start_on_prefix_line:
|
| + # If splitting after the opening bracket will cause the first element
|
| + # to be aligned weirdly, don't try it.
|
| + first_token = get_item(parsed_tokens, 0)
|
| + second_token = get_item(parsed_tokens, 1)
|
| +
|
| + if (
|
| + first_token and second_token and
|
| + unicode(second_token)[0] == '(' and
|
| + len(indentation) + len(first_token) + 1 == len(continued_indent)
|
| + ):
|
| + return None
|
| +
|
| + for item in parsed_tokens:
|
| + lines.add_space_if_needed(unicode(item), equal=True)
|
| +
|
| + save_continued_indent = continued_indent
|
| + if start_on_prefix_line and isinstance(item, Container):
|
| + start_on_prefix_line = False
|
| + continued_indent = ' ' * (lines.current_size() + 1)
|
| +
|
| + item.reflow(lines, continued_indent, break_after_open_bracket)
|
| + continued_indent = save_continued_indent
|
| +
|
| + return lines.emit()
|
| +
|
| +
|
| +def _shorten_line_at_tokens_new(tokens, source, indentation,
|
| + max_line_length):
|
| + """Shorten the line taking its length into account.
|
| +
|
| + The input is expected to be free of newlines except for inside
|
| + multiline strings and at the end.
|
| +
|
| + """
|
| + # Yield the original source so to see if it's a better choice than the
|
| + # shortened candidate lines we generate here.
|
| + yield indentation + source
|
| +
|
| + parsed_tokens = _parse_tokens(tokens)
|
| +
|
| + if parsed_tokens:
|
| + # Perform two reflows. The first one starts on the same line as the
|
| + # prefix. The second starts on the line after the prefix.
|
| + fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
|
| + start_on_prefix_line=True)
|
| + if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
|
| + yield fixed
|
| +
|
| + fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
|
| + start_on_prefix_line=False)
|
| + if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
|
| + yield fixed
|
| +
|
| +
|
| +def _shorten_line_at_tokens(tokens, source, indentation, indent_word,
|
| + key_token_strings, aggressive):
|
| + """Separate line by breaking at tokens in key_token_strings.
|
| +
|
| + The input is expected to be free of newlines except for inside
|
| + multiline strings and at the end.
|
| +
|
| + """
|
| + offsets = []
|
| + for (index, _t) in enumerate(token_offsets(tokens)):
|
| + (token_type,
|
| + token_string,
|
| + start_offset,
|
| + end_offset) = _t
|
| +
|
| + assert token_type != token.INDENT
|
| +
|
| + if token_string in key_token_strings:
|
| + # Do not break in containers with zero or one items.
|
| + unwanted_next_token = {
|
| + '(': ')',
|
| + '[': ']',
|
| + '{': '}'}.get(token_string)
|
| + if unwanted_next_token:
|
| + if (
|
| + get_item(tokens,
|
| + index + 1,
|
| + default=[None, None])[1] == unwanted_next_token or
|
| + get_item(tokens,
|
| + index + 2,
|
| + default=[None, None])[1] == unwanted_next_token
|
| + ):
|
| + continue
|
| +
|
| + if (
|
| + index > 2 and token_string == '(' and
|
| + tokens[index - 1][1] in ',(%['
|
| + ):
|
| + # Don't split after a tuple start, or before a tuple start if
|
| + # the tuple is in a list.
|
| + continue
|
| +
|
| + if end_offset < len(source) - 1:
|
| + # Don't split right before newline.
|
| + offsets.append(end_offset)
|
| + else:
|
| + # Break at adjacent strings. These were probably meant to be on
|
| + # separate lines in the first place.
|
| + previous_token = get_item(tokens, index - 1)
|
| + if (
|
| + token_type == tokenize.STRING and
|
| + previous_token and previous_token[0] == tokenize.STRING
|
| + ):
|
| + offsets.append(start_offset)
|
| +
|
| + current_indent = None
|
| + fixed = None
|
| + for line in split_at_offsets(source, offsets):
|
| + if fixed:
|
| + fixed += '\n' + current_indent + line
|
| +
|
| + for symbol in '([{':
|
| + if line.endswith(symbol):
|
| + current_indent += indent_word
|
| + else:
|
| + # First line.
|
| + fixed = line
|
| + assert not current_indent
|
| + current_indent = indent_word
|
| +
|
| + assert fixed is not None
|
| +
|
| + if check_syntax(normalize_multiline(fixed)
|
| + if aggressive > 1 else fixed):
|
| + return indentation + fixed
|
| + else:
|
| + return None
|
| +
|
| +
|
| +def token_offsets(tokens):
|
| + """Yield tokens and offsets."""
|
| + end_offset = 0
|
| + previous_end_row = 0
|
| + previous_end_column = 0
|
| + for t in tokens:
|
| + token_type = t[0]
|
| + token_string = t[1]
|
| + (start_row, start_column) = t[2]
|
| + (end_row, end_column) = t[3]
|
| +
|
| + # Account for the whitespace between tokens.
|
| + end_offset += start_column
|
| + if previous_end_row == start_row:
|
| + end_offset -= previous_end_column
|
| +
|
| + # Record the start offset of the token.
|
| + start_offset = end_offset
|
| +
|
| + # Account for the length of the token itself.
|
| + end_offset += len(token_string)
|
| +
|
| + yield (token_type,
|
| + token_string,
|
| + start_offset,
|
| + end_offset)
|
| +
|
| + previous_end_row = end_row
|
| + previous_end_column = end_column
|
| +
|
| +
|
| +def normalize_multiline(line):
|
| + """Normalize multiline-related code that will cause syntax error.
|
| +
|
| + This is for purposes of checking syntax.
|
| +
|
| + """
|
| + if line.startswith('def ') and line.rstrip().endswith(':'):
|
| + return line + ' pass'
|
| + elif line.startswith('return '):
|
| + return 'def _(): ' + line
|
| + elif line.startswith('@'):
|
| + return line + 'def _(): pass'
|
| + elif line.startswith('class '):
|
| + return line + ' pass'
|
| + elif line.startswith('if '):
|
| + return line + ' pass'
|
| + else:
|
| + return line
|
| +
|
| +
|
| +def fix_whitespace(line, offset, replacement):
|
| + """Replace whitespace at offset and return fixed line."""
|
| + # Replace escaped newlines too
|
| + left = line[:offset].rstrip('\n\r \t\\')
|
| + right = line[offset:].lstrip('\n\r \t\\')
|
| + if right.startswith('#'):
|
| + return line
|
| + else:
|
| + return left + replacement + right
|
| +
|
| +
|
| +def _execute_pep8(pep8_options, source):
|
| + """Execute pep8 via python method calls."""
|
| + class QuietReport(pep8.BaseReport):
|
| +
|
| + """Version of checker that does not print."""
|
| +
|
| + def __init__(self, options):
|
| + super(QuietReport, self).__init__(options)
|
| + self.__full_error_results = []
|
| +
|
| + def error(self, line_number, offset, text, _):
|
| + """Collect errors."""
|
| + code = super(QuietReport, self).error(line_number, offset, text, _)
|
| + if code:
|
| + self.__full_error_results.append(
|
| + {'id': code,
|
| + 'line': line_number,
|
| + 'column': offset + 1,
|
| + 'info': text})
|
| +
|
| + def full_error_results(self):
|
| + """Return error results in detail.
|
| +
|
| + Results are in the form of a list of dictionaries. Each
|
| + dictionary contains 'id', 'line', 'column', and 'info'.
|
| +
|
| + """
|
| + return self.__full_error_results
|
| +
|
| + checker = pep8.Checker('', lines=source,
|
| + reporter=QuietReport, **pep8_options)
|
| + checker.check_all()
|
| + return checker.report.full_error_results()
|
| +
|
| +
|
| +def _remove_leading_and_normalize(line):
|
| + return line.lstrip().rstrip(CR + LF) + '\n'
|
| +
|
| +
|
| +class Reindenter(object):
|
| +
|
| + """Reindents badly-indented code to uniformly use four-space indentation.
|
| +
|
| + Released to the public domain, by Tim Peters, 03 October 2000.
|
| +
|
| + """
|
| +
|
| + def __init__(self, input_text):
|
| + sio = io.StringIO(input_text)
|
| + source_lines = sio.readlines()
|
| +
|
| + self.string_content_line_numbers = multiline_string_lines(input_text)
|
| +
|
| + # File lines, rstripped & tab-expanded. Dummy at start is so
|
| + # that we can use tokenize's 1-based line numbering easily.
|
| + # Note that a line is all-blank iff it is a newline.
|
| + self.lines = []
|
| + for line_number, line in enumerate(source_lines, start=1):
|
| + # Do not modify if inside a multiline string.
|
| + if line_number in self.string_content_line_numbers:
|
| + self.lines.append(line)
|
| + else:
|
| + # Only expand leading tabs.
|
| + self.lines.append(_get_indentation(line).expandtabs() +
|
| + _remove_leading_and_normalize(line))
|
| +
|
| + self.lines.insert(0, None)
|
| + self.index = 1 # index into self.lines of next line
|
| + self.input_text = input_text
|
| +
|
| + def run(self, indent_size=DEFAULT_INDENT_SIZE):
|
| + """Fix indentation and return modified line numbers.
|
| +
|
| + Line numbers are indexed at 1.
|
| +
|
| + """
|
| + if indent_size < 1:
|
| + return self.input_text
|
| +
|
| + try:
|
| + stats = _reindent_stats(tokenize.generate_tokens(self.getline))
|
| + except (SyntaxError, tokenize.TokenError):
|
| + return self.input_text
|
| + # Remove trailing empty lines.
|
| + lines = self.lines
|
| + while lines and lines[-1] == '\n':
|
| + lines.pop()
|
| + # Sentinel.
|
| + stats.append((len(lines), 0))
|
| + # Map count of leading spaces to # we want.
|
| + have2want = {}
|
| + # Program after transformation.
|
| + after = []
|
| + # Copy over initial empty lines -- there's nothing to do until
|
| + # we see a line with *something* on it.
|
| + i = stats[0][0]
|
| + after.extend(lines[1:i])
|
| + for i in range(len(stats) - 1):
|
| + thisstmt, thislevel = stats[i]
|
| + nextstmt = stats[i + 1][0]
|
| + have = _leading_space_count(lines[thisstmt])
|
| + want = thislevel * indent_size
|
| + if want < 0:
|
| + # A comment line.
|
| + if have:
|
| + # An indented comment line. If we saw the same
|
| + # indentation before, reuse what it most recently
|
| + # mapped to.
|
| + want = have2want.get(have, -1)
|
| + if want < 0:
|
| + # Then it probably belongs to the next real stmt.
|
| + for j in range(i + 1, len(stats) - 1):
|
| + jline, jlevel = stats[j]
|
| + if jlevel >= 0:
|
| + if have == _leading_space_count(lines[jline]):
|
| + want = jlevel * indent_size
|
| + break
|
| + if want < 0: # Maybe it's a hanging
|
| + # comment like this one,
|
| + # in which case we should shift it like its base
|
| + # line got shifted.
|
| + for j in range(i - 1, -1, -1):
|
| + jline, jlevel = stats[j]
|
| + if jlevel >= 0:
|
| + want = (have + _leading_space_count(
|
| + after[jline - 1]) -
|
| + _leading_space_count(lines[jline]))
|
| + break
|
| + if want < 0:
|
| + # Still no luck -- leave it alone.
|
| + want = have
|
| + else:
|
| + want = 0
|
| + assert want >= 0
|
| + have2want[have] = want
|
| + diff = want - have
|
| + if diff == 0 or have == 0:
|
| + after.extend(lines[thisstmt:nextstmt])
|
| + else:
|
| + for line_number, line in enumerate(lines[thisstmt:nextstmt],
|
| + start=thisstmt):
|
| + if line_number in self.string_content_line_numbers:
|
| + after.append(line)
|
| + elif diff > 0:
|
| + if line == '\n':
|
| + after.append(line)
|
| + else:
|
| + after.append(' ' * diff + line)
|
| + else:
|
| + remove = min(_leading_space_count(line), -diff)
|
| + after.append(line[remove:])
|
| +
|
| + return ''.join(after)
|
| +
|
| + def getline(self):
|
| + """Line-getter for tokenize."""
|
| + if self.index >= len(self.lines):
|
| + line = ''
|
| + else:
|
| + line = self.lines[self.index]
|
| + self.index += 1
|
| + return line
|
| +
|
| +
|
| +def _reindent_stats(tokens):
|
| + """Return list of (lineno, indentlevel) pairs.
|
| +
|
| + One for each stmt and comment line. indentlevel is -1 for comment lines, as
|
| + a signal that tokenize doesn't know what to do about them; indeed, they're
|
| + our headache!
|
| +
|
| + """
|
| + find_stmt = 1 # Next token begins a fresh stmt?
|
| + level = 0 # Current indent level.
|
| + stats = []
|
| +
|
| + for t in tokens:
|
| + token_type = t[0]
|
| + sline = t[2][0]
|
| + line = t[4]
|
| +
|
| + if token_type == tokenize.NEWLINE:
|
| + # A program statement, or ENDMARKER, will eventually follow,
|
| + # after some (possibly empty) run of tokens of the form
|
| + # (NL | COMMENT)* (INDENT | DEDENT+)?
|
| + find_stmt = 1
|
| +
|
| + elif token_type == tokenize.INDENT:
|
| + find_stmt = 1
|
| + level += 1
|
| +
|
| + elif token_type == tokenize.DEDENT:
|
| + find_stmt = 1
|
| + level -= 1
|
| +
|
| + elif token_type == tokenize.COMMENT:
|
| + if find_stmt:
|
| + stats.append((sline, -1))
|
| + # But we're still looking for a new stmt, so leave
|
| + # find_stmt alone.
|
| +
|
| + elif token_type == tokenize.NL:
|
| + pass
|
| +
|
| + elif find_stmt:
|
| + # This is the first "real token" following a NEWLINE, so it
|
| + # must be the first token of the next program statement, or an
|
| + # ENDMARKER.
|
| + find_stmt = 0
|
| + if line: # Not endmarker.
|
| + stats.append((sline, level))
|
| +
|
| + return stats
|
| +
|
| +
|
| +def _leading_space_count(line):
|
| + """Return number of leading spaces in line."""
|
| + i = 0
|
| + while i < len(line) and line[i] == ' ':
|
| + i += 1
|
| + return i
|
| +
|
| +
|
| +def refactor_with_2to3(source_text, fixer_names):
|
| + """Use lib2to3 to refactor the source.
|
| +
|
| + Return the refactored source code.
|
| +
|
| + """
|
| + from lib2to3.refactor import RefactoringTool
|
| + fixers = ['lib2to3.fixes.fix_' + name for name in fixer_names]
|
| + tool = RefactoringTool(fixer_names=fixers, explicit=fixers)
|
| +
|
| + from lib2to3.pgen2 import tokenize as lib2to3_tokenize
|
| + try:
|
| + return unicode(tool.refactor_string(source_text, name=''))
|
| + except lib2to3_tokenize.TokenError:
|
| + return source_text
|
| +
|
| +
|
| +def check_syntax(code):
|
| + """Return True if syntax is okay."""
|
| + try:
|
| + return compile(code, '<string>', 'exec')
|
| + except (SyntaxError, TypeError, UnicodeDecodeError):
|
| + return False
|
| +
|
| +
|
| +def filter_results(source, results, aggressive):
|
| + """Filter out spurious reports from pep8.
|
| +
|
| + If aggressive is True, we allow possibly unsafe fixes (E711, E712).
|
| +
|
| + """
|
| + non_docstring_string_line_numbers = multiline_string_lines(
|
| + source, include_docstrings=False)
|
| + all_string_line_numbers = multiline_string_lines(
|
| + source, include_docstrings=True)
|
| +
|
| + commented_out_code_line_numbers = commented_out_code_lines(source)
|
| +
|
| + for r in results:
|
| + issue_id = r['id'].lower()
|
| +
|
| + if r['line'] in non_docstring_string_line_numbers:
|
| + if issue_id.startswith(('e1', 'e501', 'w191')):
|
| + continue
|
| +
|
| + if r['line'] in all_string_line_numbers:
|
| + if issue_id in ['e501']:
|
| + continue
|
| +
|
| + # We must offset by 1 for lines that contain the trailing contents of
|
| + # multiline strings.
|
| + if not aggressive and (r['line'] + 1) in all_string_line_numbers:
|
| + # Do not modify multiline strings in non-aggressive mode. Remove
|
| + # trailing whitespace could break doctests.
|
| + if issue_id.startswith(('w29', 'w39')):
|
| + continue
|
| +
|
| + if aggressive <= 0:
|
| + if issue_id.startswith(('e711', 'w6')):
|
| + continue
|
| +
|
| + if aggressive <= 1:
|
| + if issue_id.startswith(('e712', 'e713')):
|
| + continue
|
| +
|
| + if r['line'] in commented_out_code_line_numbers:
|
| + if issue_id.startswith(('e26', 'e501')):
|
| + continue
|
| +
|
| + yield r
|
| +
|
| +
|
| +def multiline_string_lines(source, include_docstrings=False):
|
| + """Return line numbers that are within multiline strings.
|
| +
|
| + The line numbers are indexed at 1.
|
| +
|
| + Docstrings are ignored.
|
| +
|
| + """
|
| + line_numbers = set()
|
| + previous_token_type = ''
|
| + try:
|
| + for t in generate_tokens(source):
|
| + token_type = t[0]
|
| + start_row = t[2][0]
|
| + end_row = t[3][0]
|
| +
|
| + if token_type == tokenize.STRING and start_row != end_row:
|
| + if (
|
| + include_docstrings or
|
| + previous_token_type != tokenize.INDENT
|
| + ):
|
| + # We increment by one since we want the contents of the
|
| + # string.
|
| + line_numbers |= set(range(1 + start_row, 1 + end_row))
|
| +
|
| + previous_token_type = token_type
|
| + except (SyntaxError, tokenize.TokenError):
|
| + pass
|
| +
|
| + return line_numbers
|
| +
|
| +
|
| +def commented_out_code_lines(source):
|
| + """Return line numbers of comments that are likely code.
|
| +
|
| + Commented-out code is bad practice, but modifying it just adds even more
|
| + clutter.
|
| +
|
| + """
|
| + line_numbers = []
|
| + try:
|
| + for t in generate_tokens(source):
|
| + token_type = t[0]
|
| + token_string = t[1]
|
| + start_row = t[2][0]
|
| + line = t[4]
|
| +
|
| + # Ignore inline comments.
|
| + if not line.lstrip().startswith('#'):
|
| + continue
|
| +
|
| + if token_type == tokenize.COMMENT:
|
| + stripped_line = token_string.lstrip('#').strip()
|
| + if (
|
| + ' ' in stripped_line and
|
| + '#' not in stripped_line and
|
| + check_syntax(stripped_line)
|
| + ):
|
| + line_numbers.append(start_row)
|
| + except (SyntaxError, tokenize.TokenError):
|
| + pass
|
| +
|
| + return line_numbers
|
| +
|
| +
|
| +def shorten_comment(line, max_line_length, last_comment=False):
|
| + """Return trimmed or split long comment line.
|
| +
|
| + If there are no comments immediately following it, do a text wrap.
|
| + Doing this wrapping on all comments in general would lead to jagged
|
| + comment text.
|
| +
|
| + """
|
| + assert len(line) > max_line_length
|
| + line = line.rstrip()
|
| +
|
| + # PEP 8 recommends 72 characters for comment text.
|
| + indentation = _get_indentation(line) + '# '
|
| + max_line_length = min(max_line_length,
|
| + len(indentation) + 72)
|
| +
|
| + MIN_CHARACTER_REPEAT = 5
|
| + if (
|
| + len(line) - len(line.rstrip(line[-1])) >= MIN_CHARACTER_REPEAT and
|
| + not line[-1].isalnum()
|
| + ):
|
| + # Trim comments that end with things like ---------
|
| + return line[:max_line_length] + '\n'
|
| + elif last_comment and re.match(r'\s*#+\s*\w+', line):
|
| + import textwrap
|
| + split_lines = textwrap.wrap(line.lstrip(' \t#'),
|
| + initial_indent=indentation,
|
| + subsequent_indent=indentation,
|
| + width=max_line_length,
|
| + break_long_words=False,
|
| + break_on_hyphens=False)
|
| + return '\n'.join(split_lines) + '\n'
|
| + else:
|
| + return line + '\n'
|
| +
|
| +
|
| +def normalize_line_endings(lines, newline):
|
| + """Return fixed line endings.
|
| +
|
| + All lines will be modified to use the most common line ending.
|
| +
|
| + """
|
| + return [line.rstrip('\n\r') + newline for line in lines]
|
| +
|
| +
|
| +def mutual_startswith(a, b):
|
| + return b.startswith(a) or a.startswith(b)
|
| +
|
| +
|
| +def code_match(code, select, ignore):
|
| + if ignore:
|
| + assert not isinstance(ignore, unicode)
|
| + for ignored_code in [c.strip() for c in ignore]:
|
| + if mutual_startswith(code.lower(), ignored_code.lower()):
|
| + return False
|
| +
|
| + if select:
|
| + assert not isinstance(select, unicode)
|
| + for selected_code in [c.strip() for c in select]:
|
| + if mutual_startswith(code.lower(), selected_code.lower()):
|
| + return True
|
| + return False
|
| +
|
| + return True
|
| +
|
| +
|
| +def fix_code(source, options=None):
|
| + """Return fixed source code."""
|
| + if not options:
|
| + options = parse_args([''])
|
| +
|
| + if not isinstance(source, unicode):
|
| + source = source.decode(locale.getpreferredencoding())
|
| +
|
| + sio = io.StringIO(source)
|
| + return fix_lines(sio.readlines(), options=options)
|
| +
|
| +
|
| +def fix_lines(source_lines, options, filename=''):
|
| + """Return fixed source code."""
|
| + # Transform everything to line feed. Then change them back to original
|
| + # before returning fixed source code.
|
| + original_newline = find_newline(source_lines)
|
| + tmp_source = ''.join(normalize_line_endings(source_lines, '\n'))
|
| +
|
| + # Keep a history to break out of cycles.
|
| + previous_hashes = set()
|
| +
|
| + if options.line_range:
|
| + fixed_source = apply_local_fixes(tmp_source, options)
|
| + else:
|
| + # Apply global fixes only once (for efficiency).
|
| + fixed_source = apply_global_fixes(tmp_source, options)
|
| +
|
| + passes = 0
|
| + long_line_ignore_cache = set()
|
| + while hash(fixed_source) not in previous_hashes:
|
| + if options.pep8_passes >= 0 and passes > options.pep8_passes:
|
| + break
|
| + passes += 1
|
| +
|
| + previous_hashes.add(hash(fixed_source))
|
| +
|
| + tmp_source = copy.copy(fixed_source)
|
| +
|
| + fix = FixPEP8(
|
| + filename,
|
| + options,
|
| + contents=tmp_source,
|
| + long_line_ignore_cache=long_line_ignore_cache)
|
| +
|
| + fixed_source = fix.fix()
|
| +
|
| + sio = io.StringIO(fixed_source)
|
| + return ''.join(normalize_line_endings(sio.readlines(), original_newline))
|
| +
|
| +
|
| +def fix_file(filename, options=None, output=None):
|
| + if not options:
|
| + options = parse_args([filename])
|
| +
|
| + original_source = readlines_from_file(filename)
|
| +
|
| + fixed_source = original_source
|
| +
|
| + if options.in_place or output:
|
| + encoding = detect_encoding(filename)
|
| +
|
| + if output:
|
| + output = codecs.getwriter(encoding)(output.buffer
|
| + if hasattr(output, 'buffer')
|
| + else output)
|
| +
|
| + output = LineEndingWrapper(output)
|
| +
|
| + fixed_source = fix_lines(fixed_source, options, filename=filename)
|
| +
|
| + if options.diff:
|
| + new = io.StringIO(fixed_source)
|
| + new = new.readlines()
|
| + diff = get_diff_text(original_source, new, filename)
|
| + if output:
|
| + output.write(diff)
|
| + output.flush()
|
| + else:
|
| + return diff
|
| + elif options.in_place:
|
| + fp = open_with_encoding(filename, encoding=encoding,
|
| + mode='w')
|
| + fp.write(fixed_source)
|
| + fp.close()
|
| + else:
|
| + if output:
|
| + output.write(fixed_source)
|
| + output.flush()
|
| + else:
|
| + return fixed_source
|
| +
|
| +
|
| +def global_fixes():
|
| + """Yield multiple (code, function) tuples."""
|
| + for function in globals().values():
|
| + if inspect.isfunction(function):
|
| + arguments = inspect.getargspec(function)[0]
|
| + if arguments[:1] != ['source']:
|
| + continue
|
| +
|
| + code = extract_code_from_function(function)
|
| + if code:
|
| + yield (code, function)
|
| +
|
| +
|
| +def apply_global_fixes(source, options, where='global'):
|
| + """Run global fixes on source code.
|
| +
|
| + These are fixes that only need be done once (unlike those in
|
| + FixPEP8, which are dependent on pep8).
|
| +
|
| + """
|
| + if code_match('E101', select=options.select, ignore=options.ignore):
|
| + source = reindent(source,
|
| + indent_size=options.indent_size)
|
| +
|
| + for (code, function) in global_fixes():
|
| + if code_match(code, select=options.select, ignore=options.ignore):
|
| + if options.verbose:
|
| + print('---> Applying {0} fix for {1}'.format(where,
|
| + code.upper()),
|
| + file=sys.stderr)
|
| + source = function(source,
|
| + aggressive=options.aggressive)
|
| +
|
| + source = fix_2to3(source,
|
| + aggressive=options.aggressive,
|
| + select=options.select,
|
| + ignore=options.ignore)
|
| +
|
| + return source
|
| +
|
| +
|
| +def apply_local_fixes(source, options):
|
| + """Ananologus to apply_global_fixes, but runs only those which makes sense
|
| + for the given line_range.
|
| +
|
| + Do as much as we can without breaking code.
|
| +
|
| + """
|
| + def find_ge(a, x):
|
| + """Find leftmost item greater than or equal to x."""
|
| + i = bisect.bisect_left(a, x)
|
| + if i != len(a):
|
| + return i, a[i]
|
| + return len(a) - 1, a[-1]
|
| +
|
| + def find_le(a, x):
|
| + """Find rightmost value less than or equal to x."""
|
| + i = bisect.bisect_right(a, x)
|
| + if i:
|
| + return i - 1, a[i - 1]
|
| + return 0, a[0]
|
| +
|
| + def local_fix(source, start_log, end_log,
|
| + start_lines, end_lines, indents, last_line):
|
| + """apply_global_fixes to the source between start_log and end_log.
|
| +
|
| + The subsource must be the correct syntax of a complete python program
|
| + (but all lines may share an indentation). The subsource's shared indent
|
| + is removed, fixes are applied and the indent prepended back. Taking
|
| + care to not reindent strings.
|
| +
|
| + last_line is the strict cut off (options.line_range[1]), so that
|
| + lines after last_line are not modified.
|
| +
|
| + """
|
| + if end_log < start_log:
|
| + return source
|
| +
|
| + ind = indents[start_log]
|
| + indent = _get_indentation(source[start_lines[start_log]])
|
| +
|
| + sl = slice(start_lines[start_log], end_lines[end_log] + 1)
|
| +
|
| + subsource = source[sl]
|
| + # Remove indent from subsource.
|
| + if ind:
|
| + for line_no in start_lines[start_log:end_log + 1]:
|
| + pos = line_no - start_lines[start_log]
|
| + subsource[pos] = subsource[pos][ind:]
|
| +
|
| + # Fix indentation of subsource.
|
| + fixed_subsource = apply_global_fixes(''.join(subsource),
|
| + options,
|
| + where='local')
|
| + fixed_subsource = fixed_subsource.splitlines(True)
|
| +
|
| + # Add back indent for non multi-line strings lines.
|
| + msl = multiline_string_lines(''.join(fixed_subsource),
|
| + include_docstrings=False)
|
| + for i, line in enumerate(fixed_subsource):
|
| + if not i + 1 in msl:
|
| + fixed_subsource[i] = indent + line if line != '\n' else line
|
| +
|
| + # We make a special case to look at the final line, if it's a multiline
|
| + # *and* the cut off is somewhere inside it, we take the fixed
|
| + # subset up until last_line, this assumes that the number of lines
|
| + # does not change in this multiline line.
|
| + changed_lines = len(fixed_subsource)
|
| + if (start_lines[end_log] != end_lines[end_log]
|
| + and end_lines[end_log] > last_line):
|
| + after_end = end_lines[end_log] - last_line
|
| + fixed_subsource = (fixed_subsource[:-after_end] +
|
| + source[sl][-after_end:])
|
| + changed_lines -= after_end
|
| +
|
| + options.line_range[1] = (options.line_range[0] +
|
| + changed_lines - 1)
|
| +
|
| + return (source[:start_lines[start_log]] +
|
| + fixed_subsource +
|
| + source[end_lines[end_log] + 1:])
|
| +
|
| + def is_continued_stmt(line,
|
| + continued_stmts=frozenset(['else', 'elif',
|
| + 'finally', 'except'])):
|
| + return re.split('[ :]', line.strip(), 1)[0] in continued_stmts
|
| +
|
| + assert options.line_range
|
| + start, end = options.line_range
|
| + start -= 1
|
| + end -= 1
|
| + last_line = end # We shouldn't modify lines after this cut-off.
|
| +
|
| + try:
|
| + logical = _find_logical(source)
|
| + except (SyntaxError, tokenize.TokenError):
|
| + return ''.join(source)
|
| +
|
| + if not logical[0]:
|
| + # Just blank lines, this should imply that it will become '\n' ?
|
| + return apply_global_fixes(source, options)
|
| +
|
| + start_lines, indents = zip(*logical[0])
|
| + end_lines, _ = zip(*logical[1])
|
| +
|
| + source = source.splitlines(True)
|
| +
|
| + start_log, start = find_ge(start_lines, start)
|
| + end_log, end = find_le(start_lines, end)
|
| +
|
| + # Look behind one line, if it's indented less than current indent
|
| + # then we can move to this previous line knowing that its
|
| + # indentation level will not be changed.
|
| + if (start_log > 0
|
| + and indents[start_log - 1] < indents[start_log]
|
| + and not is_continued_stmt(source[start_log - 1])):
|
| + start_log -= 1
|
| + start = start_lines[start_log]
|
| +
|
| + while start < end:
|
| +
|
| + if is_continued_stmt(source[start]):
|
| + start_log += 1
|
| + start = start_lines[start_log]
|
| + continue
|
| +
|
| + ind = indents[start_log]
|
| + for t in itertools.takewhile(lambda t: t[1][1] >= ind,
|
| + enumerate(logical[0][start_log:])):
|
| + n_log, n = start_log + t[0], t[1][0]
|
| + # start shares indent up to n.
|
| +
|
| + if n <= end:
|
| + source = local_fix(source, start_log, n_log,
|
| + start_lines, end_lines,
|
| + indents, last_line)
|
| + start_log = n_log if n == end else n_log + 1
|
| + start = start_lines[start_log]
|
| + continue
|
| +
|
| + else:
|
| + # Look at the line after end and see if allows us to reindent.
|
| + after_end_log, after_end = find_ge(start_lines, end + 1)
|
| +
|
| + if indents[after_end_log] > indents[start_log]:
|
| + start_log, start = find_ge(start_lines, start + 1)
|
| + continue
|
| +
|
| + if (indents[after_end_log] == indents[start_log]
|
| + and is_continued_stmt(source[after_end])):
|
| + # find n, the beginning of the last continued statement
|
| + # Apply fix to previous block if there is one.
|
| + only_block = True
|
| + for n, n_ind in logical[0][start_log:end_log + 1][::-1]:
|
| + if n_ind == ind and not is_continued_stmt(source[n]):
|
| + n_log = start_lines.index(n)
|
| + source = local_fix(source, start_log, n_log - 1,
|
| + start_lines, end_lines,
|
| + indents, last_line)
|
| + start_log = n_log + 1
|
| + start = start_lines[start_log]
|
| + only_block = False
|
| + break
|
| + if only_block:
|
| + end_log, end = find_le(start_lines, end - 1)
|
| + continue
|
| +
|
| + source = local_fix(source, start_log, end_log,
|
| + start_lines, end_lines,
|
| + indents, last_line)
|
| + break
|
| +
|
| + return ''.join(source)
|
| +
|
| +
|
| +def extract_code_from_function(function):
|
| + """Return code handled by function."""
|
| + if not function.__name__.startswith('fix_'):
|
| + return None
|
| +
|
| + code = re.sub('^fix_', '', function.__name__)
|
| + if not code:
|
| + return None
|
| +
|
| + try:
|
| + int(code[1:])
|
| + except ValueError:
|
| + return None
|
| +
|
| + return code
|
| +
|
| +
|
| +def create_parser():
|
| + """Return command-line parser."""
|
| + # Do import locally to be friendly to those who use autopep8 as a library
|
| + # and are supporting Python 2.6.
|
| + import argparse
|
| +
|
| + parser = argparse.ArgumentParser(description=docstring_summary(__doc__),
|
| + prog='autopep8')
|
| + parser.add_argument('--version', action='version',
|
| + version='%(prog)s ' + __version__)
|
| + parser.add_argument('-v', '--verbose', action='count', dest='verbose',
|
| + default=0,
|
| + help='print verbose messages; '
|
| + 'multiple -v result in more verbose messages')
|
| + parser.add_argument('-d', '--diff', action='store_true', dest='diff',
|
| + help='print the diff for the fixed source')
|
| + parser.add_argument('-i', '--in-place', action='store_true',
|
| + help='make changes to files in place')
|
| + parser.add_argument('-r', '--recursive', action='store_true',
|
| + help='run recursively over directories; '
|
| + 'must be used with --in-place or --diff')
|
| + parser.add_argument('-j', '--jobs', type=int, metavar='n', default=1,
|
| + help='number of parallel jobs; '
|
| + 'match CPU count if value is less than 1')
|
| + parser.add_argument('-p', '--pep8-passes', metavar='n',
|
| + default=-1, type=int,
|
| + help='maximum number of additional pep8 passes '
|
| + '(default: infinite)')
|
| + parser.add_argument('-a', '--aggressive', action='count', default=0,
|
| + help='enable non-whitespace changes; '
|
| + 'multiple -a result in more aggressive changes')
|
| + parser.add_argument('--experimental', action='store_true',
|
| + help='enable experimental fixes')
|
| + parser.add_argument('--exclude', metavar='globs',
|
| + help='exclude file/directory names that match these '
|
| + 'comma-separated globs')
|
| + parser.add_argument('--list-fixes', action='store_true',
|
| + help='list codes for fixes; '
|
| + 'used by --ignore and --select')
|
| + parser.add_argument('--ignore', metavar='errors', default='',
|
| + help='do not fix these errors/warnings '
|
| + '(default: {0})'.format(DEFAULT_IGNORE))
|
| + parser.add_argument('--select', metavar='errors', default='',
|
| + help='fix only these errors/warnings (e.g. E4,W)')
|
| + parser.add_argument('--max-line-length', metavar='n', default=79, type=int,
|
| + help='set maximum allowed line length '
|
| + '(default: %(default)s)')
|
| + parser.add_argument('--range', metavar='line', dest='line_range',
|
| + default=None, type=int, nargs=2,
|
| + help='only fix errors found within this inclusive '
|
| + 'range of line numbers (e.g. 1 99); '
|
| + 'line numbers are indexed at 1')
|
| + parser.add_argument('--indent-size', default=DEFAULT_INDENT_SIZE,
|
| + type=int, metavar='n',
|
| + help='number of spaces per indent level '
|
| + '(default %(default)s)')
|
| + parser.add_argument('files', nargs='*',
|
| + help="files to format or '-' for standard in")
|
| +
|
| + return parser
|
| +
|
| +
|
| +def parse_args(arguments):
|
| + """Parse command-line options."""
|
| + parser = create_parser()
|
| + args = parser.parse_args(arguments)
|
| +
|
| + if not args.files and not args.list_fixes:
|
| + parser.error('incorrect number of arguments')
|
| +
|
| + args.files = [decode_filename(name) for name in args.files]
|
| +
|
| + if '-' in args.files:
|
| + if len(args.files) > 1:
|
| + parser.error('cannot mix stdin and regular files')
|
| +
|
| + if args.diff:
|
| + parser.error('--diff cannot be used with standard input')
|
| +
|
| + if args.in_place:
|
| + parser.error('--in-place cannot be used with standard input')
|
| +
|
| + if args.recursive:
|
| + parser.error('--recursive cannot be used with standard input')
|
| +
|
| + if len(args.files) > 1 and not (args.in_place or args.diff):
|
| + parser.error('autopep8 only takes one filename as argument '
|
| + 'unless the "--in-place" or "--diff" args are '
|
| + 'used')
|
| +
|
| + if args.recursive and not (args.in_place or args.diff):
|
| + parser.error('--recursive must be used with --in-place or --diff')
|
| +
|
| + if args.exclude and not args.recursive:
|
| + parser.error('--exclude is only relevant when used with --recursive')
|
| +
|
| + if args.in_place and args.diff:
|
| + parser.error('--in-place and --diff are mutually exclusive')
|
| +
|
| + if args.max_line_length <= 0:
|
| + parser.error('--max-line-length must be greater than 0')
|
| +
|
| + if args.select:
|
| + args.select = args.select.split(',')
|
| +
|
| + if args.ignore:
|
| + args.ignore = args.ignore.split(',')
|
| + elif not args.select:
|
| + if args.aggressive:
|
| + # Enable everything by default if aggressive.
|
| + args.select = ['E', 'W']
|
| + else:
|
| + args.ignore = DEFAULT_IGNORE.split(',')
|
| +
|
| + if args.exclude:
|
| + args.exclude = args.exclude.split(',')
|
| + else:
|
| + args.exclude = []
|
| +
|
| + if args.jobs < 1:
|
| + # Do not import multiprocessing globally in case it is not supported
|
| + # on the platform.
|
| + import multiprocessing
|
| + args.jobs = multiprocessing.cpu_count()
|
| +
|
| + if args.jobs > 1 and not args.in_place:
|
| + parser.error('parallel jobs requires --in-place')
|
| +
|
| + if args.line_range:
|
| + if args.line_range[0] <= 0:
|
| + parser.error('--range must be positive numbers')
|
| + if args.line_range[0] > args.line_range[1]:
|
| + parser.error('First value of --range should be less than or equal '
|
| + 'to the second')
|
| +
|
| + return args
|
| +
|
| +
|
| +def decode_filename(filename):
|
| + """Return Unicode filename."""
|
| + if isinstance(filename, unicode):
|
| + return filename
|
| + else:
|
| + return filename.decode(sys.getfilesystemencoding())
|
| +
|
| +
|
| +def supported_fixes():
|
| + """Yield pep8 error codes that autopep8 fixes.
|
| +
|
| + Each item we yield is a tuple of the code followed by its
|
| + description.
|
| +
|
| + """
|
| + yield ('E101', docstring_summary(reindent.__doc__))
|
| +
|
| + instance = FixPEP8(filename=None, options=None, contents='')
|
| + for attribute in dir(instance):
|
| + code = re.match('fix_([ew][0-9][0-9][0-9])', attribute)
|
| + if code:
|
| + yield (
|
| + code.group(1).upper(),
|
| + re.sub(r'\s+', ' ',
|
| + docstring_summary(getattr(instance, attribute).__doc__))
|
| + )
|
| +
|
| + for (code, function) in sorted(global_fixes()):
|
| + yield (code.upper() + (4 - len(code)) * ' ',
|
| + re.sub(r'\s+', ' ', docstring_summary(function.__doc__)))
|
| +
|
| + for code in sorted(CODE_TO_2TO3):
|
| + yield (code.upper() + (4 - len(code)) * ' ',
|
| + re.sub(r'\s+', ' ', docstring_summary(fix_2to3.__doc__)))
|
| +
|
| +
|
| +def docstring_summary(docstring):
|
| + """Return summary of docstring."""
|
| + return docstring.split('\n')[0]
|
| +
|
| +
|
| +def line_shortening_rank(candidate, indent_word, max_line_length,
|
| + experimental=False):
|
| + """Return rank of candidate.
|
| +
|
| + This is for sorting candidates.
|
| +
|
| + """
|
| + if not candidate.strip():
|
| + return 0
|
| +
|
| + rank = 0
|
| + lines = candidate.split('\n')
|
| +
|
| + offset = 0
|
| + if (
|
| + not lines[0].lstrip().startswith('#') and
|
| + lines[0].rstrip()[-1] not in '([{'
|
| + ):
|
| + for (opening, closing) in ('()', '[]', '{}'):
|
| + # Don't penalize empty containers that aren't split up. Things like
|
| + # this "foo(\n )" aren't particularly good.
|
| + opening_loc = lines[0].find(opening)
|
| + closing_loc = lines[0].find(closing)
|
| + if opening_loc >= 0:
|
| + if closing_loc < 0 or closing_loc != opening_loc + 1:
|
| + offset = max(offset, 1 + opening_loc)
|
| +
|
| + current_longest = max(offset + len(x.strip()) for x in lines)
|
| +
|
| + rank += 4 * max(0, current_longest - max_line_length)
|
| +
|
| + rank += len(lines)
|
| +
|
| + # Too much variation in line length is ugly.
|
| + rank += 2 * standard_deviation(len(line) for line in lines)
|
| +
|
| + bad_staring_symbol = {
|
| + '(': ')',
|
| + '[': ']',
|
| + '{': '}'}.get(lines[0][-1])
|
| +
|
| + if len(lines) > 1:
|
| + if (
|
| + bad_staring_symbol and
|
| + lines[1].lstrip().startswith(bad_staring_symbol)
|
| + ):
|
| + rank += 20
|
| +
|
| + for lineno, current_line in enumerate(lines):
|
| + current_line = current_line.strip()
|
| +
|
| + if current_line.startswith('#'):
|
| + continue
|
| +
|
| + for bad_start in ['.', '%', '+', '-', '/']:
|
| + if current_line.startswith(bad_start):
|
| + rank += 100
|
| +
|
| + # Do not tolerate operators on their own line.
|
| + if current_line == bad_start:
|
| + rank += 1000
|
| +
|
| + if current_line.endswith(('(', '[', '{', '.')):
|
| + # Avoid lonely opening. They result in longer lines.
|
| + if len(current_line) <= len(indent_word):
|
| + rank += 100
|
| +
|
| + # Avoid the ugliness of ", (\n".
|
| + if (
|
| + current_line.endswith('(') and
|
| + current_line[:-1].rstrip().endswith(',')
|
| + ):
|
| + rank += 100
|
| +
|
| + # Also avoid the ugliness of "foo.\nbar"
|
| + if current_line.endswith('.'):
|
| + rank += 100
|
| +
|
| + if has_arithmetic_operator(current_line):
|
| + rank += 100
|
| +
|
| + if current_line.endswith(('%', '(', '[', '{')):
|
| + rank -= 20
|
| +
|
| + # Try to break list comprehensions at the "for".
|
| + if current_line.startswith('for '):
|
| + rank -= 50
|
| +
|
| + if current_line.endswith('\\'):
|
| + # If a line ends in \-newline, it may be part of a
|
| + # multiline string. In that case, we would like to know
|
| + # how long that line is without the \-newline. If it's
|
| + # longer than the maximum, or has comments, then we assume
|
| + # that the \-newline is an okay candidate and only
|
| + # penalize it a bit.
|
| + total_len = len(current_line)
|
| + lineno += 1
|
| + while lineno < len(lines):
|
| + total_len += len(lines[lineno])
|
| +
|
| + if lines[lineno].lstrip().startswith('#'):
|
| + total_len = max_line_length
|
| + break
|
| +
|
| + if not lines[lineno].endswith('\\'):
|
| + break
|
| +
|
| + lineno += 1
|
| +
|
| + if total_len < max_line_length:
|
| + rank += 10
|
| + else:
|
| + rank += 100 if experimental else 1
|
| +
|
| + # Prefer breaking at commas rather than colon.
|
| + if ',' in current_line and current_line.endswith(':'):
|
| + rank += 10
|
| +
|
| + rank += 10 * count_unbalanced_brackets(current_line)
|
| +
|
| + return max(0, rank)
|
| +
|
| +
|
| +def standard_deviation(numbers):
|
| + """Return standard devation."""
|
| + numbers = list(numbers)
|
| + if not numbers:
|
| + return 0
|
| + mean = sum(numbers) / len(numbers)
|
| + return (sum((n - mean) ** 2 for n in numbers) /
|
| + len(numbers)) ** .5
|
| +
|
| +
|
| +def has_arithmetic_operator(line):
|
| + """Return True if line contains any arithmetic operators."""
|
| + for operator in pep8.ARITHMETIC_OP:
|
| + if operator in line:
|
| + return True
|
| +
|
| + return False
|
| +
|
| +
|
| +def count_unbalanced_brackets(line):
|
| + """Return number of unmatched open/close brackets."""
|
| + count = 0
|
| + for opening, closing in ['()', '[]', '{}']:
|
| + count += abs(line.count(opening) - line.count(closing))
|
| +
|
| + return count
|
| +
|
| +
|
| +def split_at_offsets(line, offsets):
|
| + """Split line at offsets.
|
| +
|
| + Return list of strings.
|
| +
|
| + """
|
| + result = []
|
| +
|
| + previous_offset = 0
|
| + current_offset = 0
|
| + for current_offset in sorted(offsets):
|
| + if current_offset < len(line) and previous_offset != current_offset:
|
| + result.append(line[previous_offset:current_offset].strip())
|
| + previous_offset = current_offset
|
| +
|
| + result.append(line[current_offset:])
|
| +
|
| + return result
|
| +
|
| +
|
| +class LineEndingWrapper(object):
|
| +
|
| + r"""Replace line endings to work with sys.stdout.
|
| +
|
| + It seems that sys.stdout expects only '\n' as the line ending, no matter
|
| + the platform. Otherwise, we get repeated line endings.
|
| +
|
| + """
|
| +
|
| + def __init__(self, output):
|
| + self.__output = output
|
| +
|
| + def write(self, s):
|
| + self.__output.write(s.replace('\r\n', '\n').replace('\r', '\n'))
|
| +
|
| + def flush(self):
|
| + self.__output.flush()
|
| +
|
| +
|
| +def match_file(filename, exclude):
|
| + """Return True if file is okay for modifying/recursing."""
|
| + base_name = os.path.basename(filename)
|
| +
|
| + if base_name.startswith('.'):
|
| + return False
|
| +
|
| + for pattern in exclude:
|
| + if fnmatch.fnmatch(base_name, pattern):
|
| + return False
|
| +
|
| + if not os.path.isdir(filename) and not is_python_file(filename):
|
| + return False
|
| +
|
| + return True
|
| +
|
| +
|
| +def find_files(filenames, recursive, exclude):
|
| + """Yield filenames."""
|
| + while filenames:
|
| + name = filenames.pop(0)
|
| + if recursive and os.path.isdir(name):
|
| + for root, directories, children in os.walk(name):
|
| + filenames += [os.path.join(root, f) for f in children
|
| + if match_file(os.path.join(root, f),
|
| + exclude)]
|
| + directories[:] = [d for d in directories
|
| + if match_file(os.path.join(root, d),
|
| + exclude)]
|
| + else:
|
| + yield name
|
| +
|
| +
|
| +def _fix_file(parameters):
|
| + """Helper function for optionally running fix_file() in parallel."""
|
| + if parameters[1].verbose:
|
| + print('[file:{0}]'.format(parameters[0]), file=sys.stderr)
|
| + try:
|
| + fix_file(*parameters)
|
| + except IOError as error:
|
| + print(unicode(error), file=sys.stderr)
|
| +
|
| +
|
| +def fix_multiple_files(filenames, options, output=None):
|
| + """Fix list of files.
|
| +
|
| + Optionally fix files recursively.
|
| +
|
| + """
|
| + filenames = find_files(filenames, options.recursive, options.exclude)
|
| + if options.jobs > 1:
|
| + import multiprocessing
|
| + pool = multiprocessing.Pool(options.jobs)
|
| + pool.map(_fix_file,
|
| + [(name, options) for name in filenames])
|
| + else:
|
| + for name in filenames:
|
| + _fix_file((name, options, output))
|
| +
|
| +
|
| +def is_python_file(filename):
|
| + """Return True if filename is Python file."""
|
| + if filename.endswith('.py'):
|
| + return True
|
| +
|
| + try:
|
| + with open_with_encoding(filename) as f:
|
| + first_line = f.readlines(1)[0]
|
| + except (IOError, IndexError):
|
| + return False
|
| +
|
| + if not PYTHON_SHEBANG_REGEX.match(first_line):
|
| + return False
|
| +
|
| + return True
|
| +
|
| +
|
| +def is_probably_part_of_multiline(line):
|
| + """Return True if line is likely part of a multiline string.
|
| +
|
| + When multiline strings are involved, pep8 reports the error as being
|
| + at the start of the multiline string, which doesn't work for us.
|
| +
|
| + """
|
| + return (
|
| + '"""' in line or
|
| + "'''" in line or
|
| + line.rstrip().endswith('\\')
|
| + )
|
| +
|
| +
|
| +def main():
|
| + """Tool main."""
|
| + try:
|
| + # Exit on broken pipe.
|
| + signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
| + except AttributeError: # pragma: no cover
|
| + # SIGPIPE is not available on Windows.
|
| + pass
|
| +
|
| + try:
|
| + args = parse_args(sys.argv[1:])
|
| +
|
| + if args.list_fixes:
|
| + for code, description in sorted(supported_fixes()):
|
| + print('{code} - {description}'.format(
|
| + code=code, description=description))
|
| + return 0
|
| +
|
| + if args.files == ['-']:
|
| + assert not args.in_place
|
| +
|
| + # LineEndingWrapper is unnecessary here due to the symmetry between
|
| + # standard in and standard out.
|
| + sys.stdout.write(fix_code(sys.stdin.read(), args))
|
| + else:
|
| + if args.in_place or args.diff:
|
| + args.files = list(set(args.files))
|
| + else:
|
| + assert len(args.files) == 1
|
| + assert not args.recursive
|
| +
|
| + fix_multiple_files(args.files, args, sys.stdout)
|
| + except KeyboardInterrupt:
|
| + return 1 # pragma: no cover
|
| +
|
| +
|
| +class CachedTokenizer(object):
|
| +
|
| + """A one-element cache around tokenize.generate_tokens().
|
| +
|
| + Original code written by Ned Batchelder, in coverage.py.
|
| +
|
| + """
|
| +
|
| + def __init__(self):
|
| + self.last_text = None
|
| + self.last_tokens = None
|
| +
|
| + def generate_tokens(self, text):
|
| + """A stand-in for tokenize.generate_tokens()."""
|
| + if text != self.last_text:
|
| + string_io = io.StringIO(text)
|
| + self.last_tokens = list(
|
| + tokenize.generate_tokens(string_io.readline)
|
| + )
|
| + self.last_text = text
|
| + return self.last_tokens
|
| +
|
| +_cached_tokenizer = CachedTokenizer()
|
| +generate_tokens = _cached_tokenizer.generate_tokens
|
| +
|
| +
|
| +if __name__ == '__main__':
|
| + sys.exit(main())
|
|
|
Chromium Code Reviews
Issue 546613003:
Add a new 'format-webkitpy' command that will reformat code to the style guide. (Closed)
Base URL: svn://svn.chromium.org/blink/trunk