pylint: upgrade to 1.4.0

third_party/pylint/checkers/format.py

 # Copyright (c) 2003-2013 LOGILAB S.A. (Paris, FRANCE).
 #
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
 # Foundation; either version 2 of the License, or (at your option) any later
 # version.
 #
 # This program is distributed in the hope that it will be useful, but WITHOUT
 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 """Python code format's checker.
 
 By default try to follow Guido's style guide :
 
 http://www.python.org/doc/essays/styleguide.html
 
 Some parts of the process_token method is based from The Tab Nanny std module.
 """
 
 import keyword
 import sys
 import tokenize
+from functools import reduce # pylint: disable=redefined-builtin
 
-if not hasattr(tokenize, 'NL'):
-    raise ValueError("tokenize.NL doesn't exist -- tokenize module too old")
+import six
+from six.moves import zip, map, filter # pylint: disable=redefined-builtin
 
 from astroid import nodes
 
 from pylint.interfaces import ITokenChecker, IAstroidChecker, IRawChecker
 from pylint.checkers import BaseTokenChecker
 from pylint.checkers.utils import check_messages
 from pylint.utils import WarningScope, OPTION_RGX
 
 _CONTINUATION_BLOCK_OPENERS = ['elif', 'except', 'for', 'if', 'while', 'def', 'class']
 _KEYWORD_TOKENS = ['assert', 'del', 'elif', 'except', 'for', 'if', 'in', 'not',
@@ skipping 18 matching lines @@
 
 # Whitespace checking config constants
 _DICT_SEPARATOR = 'dict-separator'
 _TRAILING_COMMA = 'trailing-comma'
 _NO_SPACE_CHECK_CHOICES = [_TRAILING_COMMA, _DICT_SEPARATOR]
 
 MSGS = {
     'C0301': ('Line too long (%s/%s)',
               'line-too-long',
               'Used when a line is longer than a given number of characters.'),
-    'C0302': ('Too many lines in module (%s)', # was W0302
+    'C0302': ('Too many lines in module (%s/%s)', # was W0302
               'too-many-lines',
               'Used when a module has too much lines, reducing its readability.'
              ),
     'C0303': ('Trailing whitespace',
               'trailing-whitespace',
               'Used when there is whitespace between the end of a line and the '
               'newline.'),
     'C0304': ('Final newline missing',
               'missing-final-newline',
               'Used when the last line in a file is missing a newline.'),
@@ skipping 19 matching lines @@
                'superfluous-parens',
                'Used when a single item in parentheses follows an if, for, or '
                'other keyword.'),
     'C0326': ('%s space %s %s %s\n%s',
               'bad-whitespace',
               ('Used when a wrong number of spaces is used around an operator, '
                'bracket or block opener.'),
               {'old_names': [('C0323', 'no-space-after-operator'),
                              ('C0324', 'no-space-after-comma'),
                              ('C0322', 'no-space-before-operator')]}),
-    'W0331': ('Use of the <> operator',
-              'old-ne-operator',
-              'Used when the deprecated "<>" operator is used instead '
-              'of "!=".',
-              {'maxversion': (3, 0)}),
     'W0332': ('Use of "l" as long integer identifier',
               'lowercase-l-suffix',
               'Used when a lower case "l" is used to mark a long integer. You '
               'should use a upper case "L" since the letter "l" looks too much '
               'like the digit "1"',
               {'maxversion': (3, 0)}),
-    'W0333': ('Use of the `` operator',
-              'backtick',
-              'Used when the deprecated "``" (backtick) operator is used '
-              'instead  of the str() function.',
-              {'scope': WarningScope.NODE, 'maxversion': (3, 0)}),
+    'C0327': ('Mixed line endings LF and CRLF',
+              'mixed-line-endings',
+              'Used when there are mixed (LF and CRLF) newline signs in a file.'),
+    'C0328': ('Unexpected line ending format. There is \'%s\' while it should be \'%s\'.',
+              'unexpected-line-ending-format',
+              'Used when there is different newline than expected.'),
     }
 
 
 def _underline_token(token):
     length = token[3][1] - token[2][1]
     offset = token[2][1]
     return token[4] + (' ' * offset) + ('^' * length)
 
 
 def _column_distance(token1, token2):
@@ skipping 195 matching lines @@
                 position,
                 _Offsets(indentation + self._continuation_size, indentation),
                 _BeforeBlockOffsets(indentation + self._continuation_size,
                                     indentation + self._continuation_size * 2))
         elif bracket == ':':
             # If the dict key was on the same line as the open brace, the new
             # correct indent should be relative to the key instead of the
             # current indent level
             paren_align = self._cont_stack[-1].valid_outdent_offsets
             next_align = self._cont_stack[-1].valid_continuation_offsets.copy()
-            next_align[next_align.keys()[0] + self._continuation_size] = True
+            next_align_keys = list(next_align.keys())
+            next_align[next_align_keys[0] + self._continuation_size] = True
             # Note that the continuation of
             # d = {
             #       'a': 'b'
             #            'c'
             # }
             # is handled by the special-casing for hanging continued string indents.
             return _ContinuedIndent(HANGING_DICT_VALUE, bracket, position, paren_align, next_align)
         else:
             return _ContinuedIndent(
                 HANGING,
@@ skipping 44 matching lines @@
         else:
             self._cont_stack.append(
                 self._continuation_inside_bracket(token, position))
 
 
 class FormatChecker(BaseTokenChecker):
     """checks for :
     * unauthorized constructions
     * strict indentation
     * line length
-    * use of <> instead of !=
     """
 
     __implements__ = (ITokenChecker, IAstroidChecker, IRawChecker)
 
     # configuration section name
     name = 'format'
     # messages
     msgs = MSGS
     # configuration options
     # for available dict keys/values see the optik parser 'add_option' method
     options = (('max-line-length',
-                {'default' : 80, 'type' : "int", 'metavar' : '<int>',
+                {'default' : 100, 'type' : "int", 'metavar' : '<int>',
                  'help' : 'Maximum number of characters on a single line.'}),
                ('ignore-long-lines',
                 {'type': 'regexp', 'metavar': '<regexp>',
                  'default': r'^\s*(# )?<?https?://\S+>?$',
                  'help': ('Regexp for a line that is allowed to be longer than '
                           'the limit.')}),
                ('single-line-if-stmt',
                 {'default': False, 'type' : 'yn', 'metavar' : '<y_or_n>',
                  'help' : ('Allow the body of an if to be on the same '
                            'line as the test if there is no else.')}),
                ('no-space-check',
                 {'default': ','.join(_NO_SPACE_CHECK_CHOICES),
                  'type': 'multiple_choice',
                  'choices': _NO_SPACE_CHECK_CHOICES,
                  'help': ('List of optional constructs for which whitespace '
                           'checking is disabled')}),
                ('max-module-lines',
                 {'default' : 1000, 'type' : 'int', 'metavar' : '<int>',
                  'help': 'Maximum number of lines in a module'}
                ),
                ('indent-string',
                 {'default' : '    ', 'type' : "string", 'metavar' : '<string>',
                  'help' : 'String used as indentation unit. This is usually '
                           '"    " (4 spaces) or "\\t" (1 tab).'}),
                ('indent-after-paren',
                 {'type': 'int', 'metavar': '<int>', 'default': 4,
                  'help': 'Number of spaces of indent required inside a hanging '
                          ' or continued line.'}),
+               ('expected-line-ending-format',
+                {'type': 'choice', 'metavar': '<empty or LF or CRLF>', 'default': '',
+                 'choices': ['', 'LF', 'CRLF'],
+                 'help': 'Expected format of line ending, e.g. empty (any line ending), LF or CRLF.'}),
               )
 
     def __init__(self, linter=None):
         BaseTokenChecker.__init__(self, linter)
         self._lines = None
         self._visited_lines = None
         self._bracket_stack = [None]
 
     def _pop_token(self):
         self._bracket_stack.pop()
@@ skipping 34 matching lines @@
         if self._inside_brackets(':') and tokens[start][1] == 'for':
             self._pop_token()
         if tokens[start+1][1] != '(':
             return
 
         found_and_or = False
         depth = 0
         keyword_token = tokens[start][1]
         line_num = tokens[start][2][0]
 
-        for i in xrange(start, len(tokens) - 1):
+        for i in range(start, len(tokens) - 1):
             token = tokens[i]
 
             # If we hit a newline, then assume any parens were for continuation.
             if token[0] == tokenize.NL:
                 return
 
             if token[1] == '(':
                 depth += 1
             elif token[1] == ')':
                 depth -= 1
@@ skipping 105 matching lines @@
         self._check_space(tokens, i, (_MUST, _MUST))
 
     def _check_space(self, tokens, i, policies):
         def _policy_string(policy):
             if policy == _MUST:
                 return 'Exactly one', 'required'
             else:
                 return 'No', 'allowed'
 
         def _name_construct(token):
-            if tokens[i][1] == ',':
+            if token[1] == ',':
                 return 'comma'
-            elif tokens[i][1] == ':':
+            elif token[1] == ':':
                 return ':'
-            elif tokens[i][1] in '()[]{}':
+            elif token[1] in '()[]{}':
                 return 'bracket'
-            elif tokens[i][1] in ('<', '>', '<=', '>=', '!=', '=='):
+            elif token[1] in ('<', '>', '<=', '>=', '!=', '=='):
                 return 'comparison'
             else:
                 if self._inside_brackets('('):
                     return 'keyword argument assignment'
                 else:
                     return 'assignment'
 
         good_space = [True, True]
-        pairs = [(tokens[i-1], tokens[i]), (tokens[i], tokens[i+1])]
+        token = tokens[i]
+        pairs = [(tokens[i-1], token), (token, tokens[i+1])]
 
         for other_idx, (policy, token_pair) in enumerate(zip(policies, pairs)):
             if token_pair[other_idx][0] in _EOL or policy == _IGNORE:
                 continue
 
             distance = _column_distance(*token_pair)
             if distance is None:
                 continue
             good_space[other_idx] = (
                 (policy == _MUST and distance == 1) or
                 (policy == _MUST_NOT and distance == 0))
 
         warnings = []
         if not any(good_space) and policies[0] == policies[1]:
             warnings.append((policies[0], 'around'))
         else:
             for ok, policy, position in zip(good_space, policies, ('before', 'after')):
                 if not ok:
                     warnings.append((policy, position))
         for policy, position in warnings:
-            construct = _name_construct(tokens[i])
+            construct = _name_construct(token)
             count, state = _policy_string(policy)
-            self.add_message('bad-whitespace', line=tokens[i][2][0],
+            self.add_message('bad-whitespace', line=token[2][0],
                              args=(count, state, position, construct,
-                                   _underline_token(tokens[i])))
+                                   _underline_token(token)))
 
     def _inside_brackets(self, left):
         return self._bracket_stack[-1] == left
 
-    def _handle_old_ne_operator(self, tokens, i):
-        if tokens[i][1] == '<>':
-            self.add_message('old-ne-operator', line=tokens[i][2][0])
-
     def _prepare_token_dispatcher(self):
         raw = [
             (_KEYWORD_TOKENS,
              self._check_keyword_parentheses),
 
             (_OPENING_BRACKETS, self._opening_bracket),
 
             (_CLOSING_BRACKETS, self._closing_bracket),
 
             (['='], self._check_equals_spacing),
 
             (_SPACED_OPERATORS, self._check_surrounded_by_space),
 
             ([','], self._handle_comma),
 
             ([':'], self._handle_colon),
 
             (['lambda'], self._open_lambda),
 
-            (['<>'], self._handle_old_ne_operator),
             ]
 
         dispatch = {}
         for tokens, handler in raw:
             for token in tokens:
                 dispatch[token] = handler
         return dispatch
 
     def process_tokens(self, tokens):
         """process tokens and search for :
 
          _ non strict indentation (i.e. not always using the <indent> parameter as
            indent unit)
          _ too long lines (i.e. longer than <max_chars>)
          _ optionally bad construct (if given, bad_construct must be a compiled
            regular expression).
         """
         self._bracket_stack = [None]
         indents = [0]
         check_equal = False
         line_num = 0
         self._lines = {}
         self._visited_lines = {}
         token_handlers = self._prepare_token_dispatcher()
+        self._last_line_ending = None
 
         self._current_line = ContinuedLineState(tokens, self.config)
         for idx, (tok_type, token, start, _, line) in enumerate(tokens):
             if start[0] != line_num:
                 line_num = start[0]
                 # A tokenizer oddity: if an indented line contains a multi-line
                 # docstring, the line member of the INDENT token does not contain
                 # the full line; therefore we check the next token on the line.
                 if tok_type == tokenize.INDENT:
                     self.new_line(TokenWrapper(tokens), idx-1, idx+1)
                 else:
                     self.new_line(TokenWrapper(tokens), idx-1, idx)
 
             if tok_type == tokenize.NEWLINE:
                 # a program statement, or ENDMARKER, will eventually follow,
                 # after some (possibly empty) run of tokens of the form
                 #     (NL | COMMENT)* (INDENT | DEDENT+)?
                 # If an INDENT appears, setting check_equal is wrong, and will
                 # be undone when we see the INDENT.
                 check_equal = True
                 self._process_retained_warnings(TokenWrapper(tokens), idx)
                 self._current_line.next_logical_line()
+                self._check_line_ending(token, line_num)
             elif tok_type == tokenize.INDENT:
                 check_equal = False
                 self.check_indent_level(token, indents[-1]+1, line_num)
                 indents.append(indents[-1]+1)
             elif tok_type == tokenize.DEDENT:
                 # there's nothing we need to check here!  what's important is
                 # that when the run of DEDENTs ends, the indentation of the
                 # program statement (or ENDMARKER) that triggered the run is
                 # equal to what's left at the top of the indents stack
                 check_equal = True
@@ skipping 19 matching lines @@
 
             try:
                 handler = token_handlers[token]
             except KeyError:
                 pass
             else:
                 handler(tokens, idx)
 
         line_num -= 1 # to be ok with "wc -l"
         if line_num > self.config.max_module_lines:
-            self.add_message('too-many-lines', args=line_num, line=1)
+            # Get the line where the too-many-lines (or its message id)
+            # was disabled or default to 1.
+            symbol = self.linter.msgs_store.check_message_id('too-many-lines')
+            names = (symbol.msgid, 'too-many-lines')
+            line = next(filter(None,
+                               map(self.linter._pragma_lineno.get, names)), 1)
+            self.add_message('too-many-lines',
+                             args=(line_num, self.config.max_module_lines),
+                             line=line)
+
+    def _check_line_ending(self, line_ending, line_num):
+        # check if line endings are mixed
+        if self._last_line_ending is not None:
+            if line_ending != self._last_line_ending:
+                self.add_message('mixed-line-endings', line=line_num)
+
+        self._last_line_ending = line_ending
+
+        # check if line ending is as expected
+        expected = self.config.expected_line_ending_format
+        if expected:
+            line_ending = reduce(lambda x, y: x + y if x != y else x, line_ending, "")  # reduce multiple \n\n\n\n to one \n
+            line_ending = 'LF' if line_ending == '\n' else 'CRLF'
+            if line_ending != expected:
+                self.add_message('unexpected-line-ending-format', args=(line_ending, expected), line=line_num)
+
 
     def _process_retained_warnings(self, tokens, current_pos):
         single_line_block_stmt = not _last_token_on_line_is(tokens, current_pos, ':')
 
         for indent_pos, state, offsets in self._current_line.retained_warnings:
             block_type = offsets[tokens.start_col(indent_pos)]
-            hints = dict((k, v) for k, v in offsets.iteritems()
+            hints = dict((k, v) for k, v in six.iteritems(offsets)
                          if v != block_type)
             if single_line_block_stmt and block_type == WITH_BODY:
                 self._add_continuation_message(state, hints, tokens, indent_pos)
             elif not single_line_block_stmt and block_type == SINGLE_LINE:
                 self._add_continuation_message(state, hints, tokens, indent_pos)
 
     def _check_continued_indentation(self, tokens, next_idx):
         def same_token_around_nl(token_type):
             return (tokens.type(next_idx) == token_type and
                     tokens.type(next_idx-2) == token_type)
@@ skipping 53 matching lines @@
             self._check_multi_statement_line(node, line)
             return
         if line in self._visited_lines:
             return
         try:
             tolineno = node.blockstart_tolineno
         except AttributeError:
             tolineno = node.tolineno
         assert tolineno, node
         lines = []
-        for line in xrange(line, tolineno + 1):
+        for line in range(line, tolineno + 1):
             self._visited_lines[line] = 1
             try:
                 lines.append(self._lines[line].rstrip())
             except KeyError:
                 lines.append('')
 
     def _check_multi_statement_line(self, node, line):
         """Check for lines containing multiple statements."""
         # Do not warn about multiple nested context managers
         # in with statements.
         if isinstance(node, nodes.With):
             return
         # For try... except... finally..., the two nodes
         # appear to be on the same line due to how the AST is built.
         if (isinstance(node, nodes.TryExcept) and
                 isinstance(node.parent, nodes.TryFinally)):
             return
         if (isinstance(node.parent, nodes.If) and not node.parent.orelse
                 and self.config.single_line_if_stmt):
             return
         self.add_message('multiple-statements', node=node)
         self._visited_lines[line] = 2
 
-    @check_messages('backtick')
-    def visit_backquote(self, node):
-        self.add_message('backtick', node=node)
-
     def check_lines(self, lines, i):
         """check lines have less than a maximum number of characters
         """
         max_chars = self.config.max_line_length
         ignore_long_line = self.config.ignore_long_lines
 
         for line in lines.splitlines(True):
             if not line.endswith('\n'):
                 self.add_message('missing-final-newline', line=i)
             else:
@@ skipping 37 matching lines @@
             if indent[0] == '\t':
                 i_type = 'tabs'
             self.add_message('bad-indentation', line=line_num,
                              args=(level * unit_size + len(suppl), i_type,
                                    expected * unit_size))
 
 
 def register(linter):
     """required method to auto register this checker """
     linter.register_checker(FormatChecker(linter))