binding: Updates Jinja2 from 2.7.1 to 2.8.

third_party/jinja2/lexer.py

 # -*- coding: utf-8 -*-
 """
     jinja2.lexer
     ~~~~~~~~~~~~
 
     This module implements a Jinja / Python combination lexer. The
     `Lexer` class provided by this module is used to do some preprocessing
     for Jinja.
 
     On the one hand it filters out invalid operators like the bitshift
     operators we don't allow in templates. On the other hand it separates
     template code and python code in expressions.
 
     :copyright: (c) 2010 by the Jinja Team.
     :license: BSD, see LICENSE for more details.
 """
 import re
 
 from operator import itemgetter
 from collections import deque
 from jinja2.exceptions import TemplateSyntaxError
 from jinja2.utils import LRUCache
-from jinja2._compat import next, iteritems, implements_iterator, text_type, \
-     intern
+from jinja2._compat import iteritems, implements_iterator, text_type, \
+     intern, PY2
 
 
 # cache for the lexers. Exists in order to be able to have multiple
 # environments with the same lexer
 _lexer_cache = LRUCache(50)
 
 # static regular expressions
 whitespace_re = re.compile(r'\s+', re.U)
 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                        r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
@@ skipping 94 matching lines @@
     ';':            TOKEN_SEMICOLON
 }
 
 reverse_operators = dict([(v, k) for k, v in iteritems(operators)])
 assert len(operators) == len(reverse_operators), 'operators dropped'
 operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
                          sorted(operators, key=lambda x: -len(x))))
 
 ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
                             TOKEN_COMMENT_END, TOKEN_WHITESPACE,
-                            TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN,
-                            TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT])
+                            TOKEN_LINECOMMENT_BEGIN, TOKEN_LINECOMMENT_END,
+                            TOKEN_LINECOMMENT])
 ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA,
                              TOKEN_COMMENT, TOKEN_LINECOMMENT])
 
 
 def _describe_token_type(token_type):
     if token_type in reverse_operators:
         return reverse_operators[token_type]
     return {
         TOKEN_COMMENT_BEGIN:        'begin of comment',
         TOKEN_COMMENT_END:          'end of comment',
@@ skipping 420 matching lines @@
                     value = self._normalize_newlines(value[1:-1]) \
                         .encode('ascii', 'backslashreplace') \
                         .decode('unicode-escape')
                 except Exception as e:
                     msg = str(e).split(':')[-1].strip()
                     raise TemplateSyntaxError(msg, lineno, name, filename)
                 # if we can express it as bytestring (ascii only)
                 # we do that for support of semi broken APIs
                 # as datetime.datetime.strftime.  On python 3 this
                 # call becomes a noop thanks to 2to3
-                try:
-                    value = str(value)
-                except UnicodeError:
-                    pass
+                if PY2:
+                    try:
+                        value = value.encode('ascii')
+                    except UnicodeError:
+                        pass
             elif token == 'integer':
                 value = int(value)
             elif token == 'float':
                 value = float(value)
             elif token == 'operator':
                 token = operators[value]
             yield Token(lineno, token, value)
 
     def tokeniter(self, source, name, filename=None, state=None):
         """This method tokenizes the text and returns the tokens in a
@@ skipping 129 matching lines @@
             # if loop terminated without break we haven't found a single match
             # either we are at the end of the file or we have a problem
             else:
                 # end of text
                 if pos >= source_length:
                     return
                 # something went wrong
                 raise TemplateSyntaxError('unexpected char %r at %d' %
                                           (source[pos], pos), lineno,
                                           name, filename)