Update Jinja2 (Python template library) to 2.7.1

third_party/jinja2/lexer.py

 # -*- coding: utf-8 -*-
 """
     jinja2.lexer
     ~~~~~~~~~~~~
 
     This module implements a Jinja / Python combination lexer. The
     `Lexer` class provided by this module is used to do some preprocessing
     for Jinja.
 
     On the one hand it filters out invalid operators like the bitshift
     operators we don't allow in templates. On the other hand it separates
     template code and python code in expressions.
 
     :copyright: (c) 2010 by the Jinja Team.
     :license: BSD, see LICENSE for more details.
 """
 import re
+
 from operator import itemgetter
 from collections import deque
 from jinja2.exceptions import TemplateSyntaxError
-from jinja2.utils import LRUCache, next
+from jinja2.utils import LRUCache
+from jinja2._compat import next, iteritems, implements_iterator, text_type, \
+     intern
 
 
 # cache for the lexers. Exists in order to be able to have multiple
 # environments with the same lexer
 _lexer_cache = LRUCache(50)
 
 # static regular expressions
 whitespace_re = re.compile(r'\s+', re.U)
 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                        r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
@@ skipping 87 matching lines @@
     '<':            TOKEN_LT,
     '<=':           TOKEN_LTEQ,
     '=':            TOKEN_ASSIGN,
     '.':            TOKEN_DOT,
     ':':            TOKEN_COLON,
     '|':            TOKEN_PIPE,
     ',':            TOKEN_COMMA,
     ';':            TOKEN_SEMICOLON
 }
 
-reverse_operators = dict([(v, k) for k, v in operators.iteritems()])
+reverse_operators = dict([(v, k) for k, v in iteritems(operators)])
 assert len(operators) == len(reverse_operators), 'operators dropped'
 operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
                          sorted(operators, key=lambda x: -len(x))))
 
 ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
                             TOKEN_COMMENT_END, TOKEN_WHITESPACE,
                             TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN,
                             TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT])
 ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA,
                              TOKEN_COMMENT, TOKEN_LINECOMMENT])
@@ skipping 50 matching lines @@
         (len(environment.comment_start_string), 'comment',
          e(environment.comment_start_string)),
         (len(environment.block_start_string), 'block',
          e(environment.block_start_string)),
         (len(environment.variable_start_string), 'variable',
          e(environment.variable_start_string))
     ]
 
     if environment.line_statement_prefix is not None:
         rules.append((len(environment.line_statement_prefix), 'linestatement',
-                      r'^\s*' + e(environment.line_statement_prefix)))
+                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
     if environment.line_comment_prefix is not None:
         rules.append((len(environment.line_comment_prefix), 'linecomment',
                       r'(?:^|(?<=\S))[^\S\r\n]*' +
                       e(environment.line_comment_prefix)))
 
     return [x[1:] for x in sorted(rules, reverse=True)]
 
 
 class Failure(object):
     """Class that raises a `TemplateSyntaxError` if called.
@@ skipping 44 matching lines @@
         return False
 
     def __repr__(self):
         return 'Token(%r, %r, %r)' % (
             self.lineno,
             self.type,
             self.value
         )
 
 
+@implements_iterator
 class TokenStreamIterator(object):
     """The iterator for tokenstreams.  Iterate over the stream
     until the eof token is reached.
     """
 
     def __init__(self, stream):
         self.stream = stream
 
     def __iter__(self):
         return self
 
-    def next(self):
+    def __next__(self):
         token = self.stream.current
         if token.type is TOKEN_EOF:
             self.stream.close()
             raise StopIteration()
         next(self.stream)
         return token
 
 
+@implements_iterator
 class TokenStream(object):
     """A token stream is an iterable that yields :class:`Token`\s.  The
     parser however does not iterate over it but calls :meth:`next` to go
     one token ahead.  The current active token is stored as :attr:`current`.
     """
 
     def __init__(self, generator, name, filename):
-        self._next = iter(generator).next
+        self._iter = iter(generator)
         self._pushed = deque()
         self.name = name
         self.filename = filename
         self.closed = False
         self.current = Token(1, TOKEN_INITIAL, '')
         next(self)
 
     def __iter__(self):
         return TokenStreamIterator(self)
 
-    def __nonzero__(self):
+    def __bool__(self):
         return bool(self._pushed) or self.current.type is not TOKEN_EOF
+    __nonzero__ = __bool__  # py2
 
     eos = property(lambda x: not x, doc="Are we at the end of the stream?")
 
     def push(self, token):
         """Push a token back to the stream."""
         self._pushed.append(token)
 
     def look(self):
         """Look at the next token."""
         old_token = next(self)
         result = self.current
         self.push(result)
         self.current = old_token
         return result
 
     def skip(self, n=1):
         """Got n tokens ahead."""
-        for x in xrange(n):
+        for x in range(n):
             next(self)
 
     def next_if(self, expr):
         """Perform the token test and return the token if it matched.
         Otherwise the return value is `None`.
         """
         if self.current.test(expr):
             return next(self)
 
     def skip_if(self, expr):
         """Like :meth:`next_if` but only returns `True` or `False`."""
         return self.next_if(expr) is not None
 
-    def next(self):
+    def __next__(self):
         """Go one token ahead and return the old one"""
         rv = self.current
         if self._pushed:
             self.current = self._pushed.popleft()
         elif self.current.type is not TOKEN_EOF:
             try:
-                self.current = self._next()
+                self.current = next(self._iter)
             except StopIteration:
                 self.close()
         return rv
 
     def close(self):
         """Close the stream."""
         self.current = Token(self.current.lineno, TOKEN_EOF, '')
-        self._next = None
+        self._iter = None
         self.closed = True
 
     def expect(self, expr):
         """Expect a given token type and return it.  This accepts the same
         argument as :meth:`jinja2.lexer.Token.test`.
         """
         if not self.current.test(expr):
             expr = describe_token_expr(expr)
             if self.current.type is TOKEN_EOF:
                 raise TemplateSyntaxError('unexpected end of template, '
@@ skipping 14 matching lines @@
     """Return a lexer which is probably cached."""
     key = (environment.block_start_string,
            environment.block_end_string,
            environment.variable_start_string,
            environment.variable_end_string,
            environment.comment_start_string,
            environment.comment_end_string,
            environment.line_statement_prefix,
            environment.line_comment_prefix,
            environment.trim_blocks,
-           environment.newline_sequence)
+           environment.lstrip_blocks,
+           environment.newline_sequence,
+           environment.keep_trailing_newline)
     lexer = _lexer_cache.get(key)
     if lexer is None:
         lexer = Lexer(environment)
         _lexer_cache[key] = lexer
     return lexer
 
 
 class Lexer(object):
     """Class that implements a lexer for a given environment. Automatically
     created by the environment class, usually you don't have to do that.
@@ skipping 10 matching lines @@
         # lexing rules for tags
         tag_rules = [
             (whitespace_re, TOKEN_WHITESPACE, None),
             (float_re, TOKEN_FLOAT, None),
             (integer_re, TOKEN_INTEGER, None),
             (name_re, TOKEN_NAME, None),
             (string_re, TOKEN_STRING, None),
             (operator_re, TOKEN_OPERATOR, None)
         ]
 
-        # assamble the root lexing rule. because "|" is ungreedy
+        # assemble the root lexing rule. because "|" is ungreedy
         # we have to sort by length so that the lexer continues working
         # as expected when we have parsing rules like <% for block and
         # <%= for variables. (if someone wants asp like syntax)
         # variables are just part of the rules if variable processing
         # is required.
         root_tag_rules = compile_rules(environment)
 
         # block suffix if trimming is enabled
         block_suffix_re = environment.trim_blocks and '\\n?' or ''
 
+        # strip leading spaces if lstrip_blocks is enabled
+        prefix_re = {}
+        if environment.lstrip_blocks:
+            # use '{%+' to manually disable lstrip_blocks behavior
+            no_lstrip_re = e('+')
+            # detect overlap between block and variable or comment strings
+            block_diff = c(r'^%s(.*)' % e(environment.block_start_string))
+            # make sure we don't mistake a block for a variable or a comment
+            m = block_diff.match(environment.comment_start_string)
+            no_lstrip_re += m and r'|%s' % e(m.group(1)) or ''
+            m = block_diff.match(environment.variable_start_string)
+            no_lstrip_re += m and r'|%s' % e(m.group(1)) or ''
+
+            # detect overlap between comment and variable strings
+            comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string))
+            m = comment_diff.match(environment.variable_start_string)
+            no_variable_re = m and r'(?!%s)' % e(m.group(1)) or ''
+
+            lstrip_re = r'^[ \t]*'
+            block_prefix_re = r'%s%s(?!%s)|%s\+?' % (
+                    lstrip_re,
+                    e(environment.block_start_string),
+                    no_lstrip_re,
+                    e(environment.block_start_string),
+                    )
+            comment_prefix_re = r'%s%s%s|%s\+?' % (
+                    lstrip_re,
+                    e(environment.comment_start_string),
+                    no_variable_re,
+                    e(environment.comment_start_string),
+                    )
+            prefix_re['block'] = block_prefix_re
+            prefix_re['comment'] = comment_prefix_re
+        else:
+            block_prefix_re = '%s' % e(environment.block_start_string)
+
         self.newline_sequence = environment.newline_sequence
+        self.keep_trailing_newline = environment.keep_trailing_newline
 
         # global lexing rules
         self.rules = {
             'root': [
                 # directives
                 (c('(.*?)(?:%s)' % '|'.join(
                     [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
                         e(environment.block_start_string),
-                        e(environment.block_start_string),
+                        block_prefix_re,
                         e(environment.block_end_string),
                         e(environment.block_end_string)
                     )] + [
-                        r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r)
+                        r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r))
                         for n, r in root_tag_rules
                     ])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
                 # data
                 (c('.+'), TOKEN_DATA, None)
             ],
             # comments
             TOKEN_COMMENT_BEGIN: [
                 (c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
                     e(environment.comment_end_string),
                     e(environment.comment_end_string),
@@ skipping 13 matching lines @@
             TOKEN_VARIABLE_BEGIN: [
                 (c('\-%s\s*|%s' % (
                     e(environment.variable_end_string),
                     e(environment.variable_end_string)
                 )), TOKEN_VARIABLE_END, '#pop')
             ] + tag_rules,
             # raw block
             TOKEN_RAW_BEGIN: [
                 (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
                     e(environment.block_start_string),
-                    e(environment.block_start_string),
+                    block_prefix_re,
                     e(environment.block_end_string),
                     e(environment.block_end_string),
                     block_suffix_re
                 )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
                 (c('(.)'), (Failure('Missing end of raw directive'),), None)
             ],
             # line statements
             TOKEN_LINESTATEMENT_BEGIN: [
                 (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
             ] + tag_rules,
             # line comments
             TOKEN_LINECOMMENT_BEGIN: [
                 (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT,
                  TOKEN_LINECOMMENT_END), '#pop')
             ]
         }
 
     def _normalize_newlines(self, value):
-        """Called for strings and template data to normlize it to unicode."""
+        """Called for strings and template data to normalize it to unicode."""
         return newline_re.sub(self.newline_sequence, value)
 
     def tokenize(self, source, name=None, filename=None, state=None):
         """Calls tokeniter + tokenize and wraps it in a token stream.
         """
         stream = self.tokeniter(source, name, filename, state)
         return TokenStream(self.wrap(stream, name, filename), name, filename)
 
     def wrap(self, stream, name=None, filename=None):
         """This is called with the stream as returned by `tokenize` and wraps
@@ skipping 14 matching lines @@
             elif token == 'keyword':
                 token = value
             elif token == 'name':
                 value = str(value)
             elif token == 'string':
                 # try to unescape string
                 try:
                     value = self._normalize_newlines(value[1:-1]) \
                         .encode('ascii', 'backslashreplace') \
                         .decode('unicode-escape')
-                except Exception, e:
+                except Exception as e:
                     msg = str(e).split(':')[-1].strip()
                     raise TemplateSyntaxError(msg, lineno, name, filename)
                 # if we can express it as bytestring (ascii only)
                 # we do that for support of semi broken APIs
                 # as datetime.datetime.strftime.  On python 3 this
                 # call becomes a noop thanks to 2to3
                 try:
                     value = str(value)
                 except UnicodeError:
                     pass
             elif token == 'integer':
                 value = int(value)
             elif token == 'float':
                 value = float(value)
             elif token == 'operator':
                 token = operators[value]
             yield Token(lineno, token, value)
 
     def tokeniter(self, source, name, filename=None, state=None):
         """This method tokenizes the text and returns the tokens in a
         generator.  Use this method if you just want to tokenize a template.
         """
-        source = '\n'.join(unicode(source).splitlines())
+        source = text_type(source)
+        lines = source.splitlines()
+        if self.keep_trailing_newline and source:
+            for newline in ('\r\n', '\r', '\n'):
+                if source.endswith(newline):
+                    lines.append('')
+                    break
+        source = '\n'.join(lines)
         pos = 0
         lineno = 1
         stack = ['root']
         if state is not None and state != 'root':
             assert state in ('variable', 'block'), 'invalid state'
             stack.append(state + '_begin')
         else:
             state = 'root'
         statetokens = self.rules[stack[-1]]
         source_length = len(source)
 
         balancing_stack = []
 
         while 1:
             # tokenizer loop
             for regex, tokens, new_state in statetokens:
                 m = regex.match(source, pos)
                 # if no match we try again with the next rule
                 if m is None:
                     continue
 
-                # we only match blocks and variables if brances / parentheses
+                # we only match blocks and variables if braces / parentheses
                 # are balanced. continue parsing with the lower rule which
                 # is the operator rule. do this only if the end tags look
                 # like operators
                 if balancing_stack and \
                    tokens in ('variable_end', 'block_end',
                               'linestatement_end'):
                     continue
 
                 # tuples support more options
                 if isinstance(tokens, tuple):
                     for idx, token in enumerate(tokens):
                         # failure group
                         if token.__class__ is Failure:
                             raise token(lineno, filename)
                         # bygroup is a bit more complex, in that case we
                         # yield for the current token the first named
                         # group that matched
                         elif token == '#bygroup':
-                            for key, value in m.groupdict().iteritems():
+                            for key, value in iteritems(m.groupdict()):
                                 if value is not None:
                                     yield lineno, key, value
                                     lineno += value.count('\n')
                                     break
                             else:
                                 raise RuntimeError('%r wanted to resolve '
                                                    'the token dynamically'
                                                    ' but no group matched'
                                                    % regex)
                         # normal group
@@ skipping 36 matching lines @@
                 # in an infinite loop
                 pos2 = m.end()
 
                 # handle state changes
                 if new_state is not None:
                     # remove the uppermost state
                     if new_state == '#pop':
                         stack.pop()
                     # resolve the new state by group checking
                     elif new_state == '#bygroup':
-                        for key, value in m.groupdict().iteritems():
+                        for key, value in iteritems(m.groupdict()):
                             if value is not None:
                                 stack.append(key)
                                 break
                         else:
                             raise RuntimeError('%r wanted to resolve the '
                                                'new state dynamically but'
                                                ' no group matched' %
                                                regex)
                     # direct state name given
                     else:
                         stack.append(new_state)
                     statetokens = self.rules[stack[-1]]
                 # we are still at the same position and no stack change.
                 # this means a loop without break condition, avoid that and
                 # raise error
                 elif pos2 == pos:
                     raise RuntimeError('%r yielded empty string without '
                                        'stack change' % regex)
                 # publish new function and start again
                 pos = pos2
                 break
-            # if loop terminated without break we havn't found a single match
+            # if loop terminated without break we haven't found a single match
             # either we are at the end of the file or we have a problem
             else:
                 # end of text
                 if pos >= source_length:
                     return
                 # something went wrong
                 raise TemplateSyntaxError('unexpected char %r at %d' %
                                           (source[pos], pos), lineno,
                                           name, filename)