| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- coding: utf-8 -*- | |
| 2 """ | |
| 3 jinja2.lexer | |
| 4 ~~~~~~~~~~~~ | |
| 5 | |
| 6 This module implements a Jinja / Python combination lexer. The | |
| 7 `Lexer` class provided by this module is used to do some preprocessing | |
| 8 for Jinja. | |
| 9 | |
| 10 On the one hand it filters out invalid operators like the bitshift | |
| 11 operators we don't allow in templates. On the other hand it separates | |
| 12 template code and python code in expressions. | |
| 13 | |
| 14 :copyright: (c) 2010 by the Jinja Team. | |
| 15 :license: BSD, see LICENSE for more details. | |
| 16 """ | |
| 17 import re | |
| 18 | |
| 19 from operator import itemgetter | |
| 20 from collections import deque | |
| 21 from jinja2.exceptions import TemplateSyntaxError | |
| 22 from jinja2.utils import LRUCache | |
| 23 from jinja2._compat import next, iteritems, implements_iterator, text_type, \ | |
| 24 intern | |
| 25 | |
| 26 | |
| 27 # cache for the lexers. Exists in order to be able to have multiple | |
| 28 # environments with the same lexer | |
| 29 _lexer_cache = LRUCache(50) | |
| 30 | |
| 31 # static regular expressions | |
| 32 whitespace_re = re.compile(r'\s+', re.U) | |
| 33 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" | |
| 34 r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) | |
| 35 integer_re = re.compile(r'\d+') | |
| 36 | |
| 37 # we use the unicode identifier rule if this python version is able | |
| 38 # to handle unicode identifiers, otherwise the standard ASCII one. | |
| 39 try: | |
| 40 compile('föö', '<unknown>', 'eval') | |
| 41 except SyntaxError: | |
| 42 name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b') | |
| 43 else: | |
| 44 from jinja2 import _stringdefs | |
| 45 name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start, | |
| 46 _stringdefs.xid_continue)) | |
| 47 | |
| 48 float_re = re.compile(r'(?<!\.)\d+\.\d+') | |
| 49 newline_re = re.compile(r'(\r\n|\r|\n)') | |
| 50 | |
| 51 # internal the tokens and keep references to them | |
| 52 TOKEN_ADD = intern('add') | |
| 53 TOKEN_ASSIGN = intern('assign') | |
| 54 TOKEN_COLON = intern('colon') | |
| 55 TOKEN_COMMA = intern('comma') | |
| 56 TOKEN_DIV = intern('div') | |
| 57 TOKEN_DOT = intern('dot') | |
| 58 TOKEN_EQ = intern('eq') | |
| 59 TOKEN_FLOORDIV = intern('floordiv') | |
| 60 TOKEN_GT = intern('gt') | |
| 61 TOKEN_GTEQ = intern('gteq') | |
| 62 TOKEN_LBRACE = intern('lbrace') | |
| 63 TOKEN_LBRACKET = intern('lbracket') | |
| 64 TOKEN_LPAREN = intern('lparen') | |
| 65 TOKEN_LT = intern('lt') | |
| 66 TOKEN_LTEQ = intern('lteq') | |
| 67 TOKEN_MOD = intern('mod') | |
| 68 TOKEN_MUL = intern('mul') | |
| 69 TOKEN_NE = intern('ne') | |
| 70 TOKEN_PIPE = intern('pipe') | |
| 71 TOKEN_POW = intern('pow') | |
| 72 TOKEN_RBRACE = intern('rbrace') | |
| 73 TOKEN_RBRACKET = intern('rbracket') | |
| 74 TOKEN_RPAREN = intern('rparen') | |
| 75 TOKEN_SEMICOLON = intern('semicolon') | |
| 76 TOKEN_SUB = intern('sub') | |
| 77 TOKEN_TILDE = intern('tilde') | |
| 78 TOKEN_WHITESPACE = intern('whitespace') | |
| 79 TOKEN_FLOAT = intern('float') | |
| 80 TOKEN_INTEGER = intern('integer') | |
| 81 TOKEN_NAME = intern('name') | |
| 82 TOKEN_STRING = intern('string') | |
| 83 TOKEN_OPERATOR = intern('operator') | |
| 84 TOKEN_BLOCK_BEGIN = intern('block_begin') | |
| 85 TOKEN_BLOCK_END = intern('block_end') | |
| 86 TOKEN_VARIABLE_BEGIN = intern('variable_begin') | |
| 87 TOKEN_VARIABLE_END = intern('variable_end') | |
| 88 TOKEN_RAW_BEGIN = intern('raw_begin') | |
| 89 TOKEN_RAW_END = intern('raw_end') | |
| 90 TOKEN_COMMENT_BEGIN = intern('comment_begin') | |
| 91 TOKEN_COMMENT_END = intern('comment_end') | |
| 92 TOKEN_COMMENT = intern('comment') | |
| 93 TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') | |
| 94 TOKEN_LINESTATEMENT_END = intern('linestatement_end') | |
| 95 TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') | |
| 96 TOKEN_LINECOMMENT_END = intern('linecomment_end') | |
| 97 TOKEN_LINECOMMENT = intern('linecomment') | |
| 98 TOKEN_DATA = intern('data') | |
| 99 TOKEN_INITIAL = intern('initial') | |
| 100 TOKEN_EOF = intern('eof') | |
| 101 | |
| 102 # bind operators to token types | |
| 103 operators = { | |
| 104 '+': TOKEN_ADD, | |
| 105 '-': TOKEN_SUB, | |
| 106 '/': TOKEN_DIV, | |
| 107 '//': TOKEN_FLOORDIV, | |
| 108 '*': TOKEN_MUL, | |
| 109 '%': TOKEN_MOD, | |
| 110 '**': TOKEN_POW, | |
| 111 '~': TOKEN_TILDE, | |
| 112 '[': TOKEN_LBRACKET, | |
| 113 ']': TOKEN_RBRACKET, | |
| 114 '(': TOKEN_LPAREN, | |
| 115 ')': TOKEN_RPAREN, | |
| 116 '{': TOKEN_LBRACE, | |
| 117 '}': TOKEN_RBRACE, | |
| 118 '==': TOKEN_EQ, | |
| 119 '!=': TOKEN_NE, | |
| 120 '>': TOKEN_GT, | |
| 121 '>=': TOKEN_GTEQ, | |
| 122 '<': TOKEN_LT, | |
| 123 '<=': TOKEN_LTEQ, | |
| 124 '=': TOKEN_ASSIGN, | |
| 125 '.': TOKEN_DOT, | |
| 126 ':': TOKEN_COLON, | |
| 127 '|': TOKEN_PIPE, | |
| 128 ',': TOKEN_COMMA, | |
| 129 ';': TOKEN_SEMICOLON | |
| 130 } | |
| 131 | |
| 132 reverse_operators = dict([(v, k) for k, v in iteritems(operators)]) | |
| 133 assert len(operators) == len(reverse_operators), 'operators dropped' | |
| 134 operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in | |
| 135 sorted(operators, key=lambda x: -len(x)))) | |
| 136 | |
| 137 ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT, | |
| 138 TOKEN_COMMENT_END, TOKEN_WHITESPACE, | |
| 139 TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN, | |
| 140 TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT]) | |
| 141 ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA, | |
| 142 TOKEN_COMMENT, TOKEN_LINECOMMENT]) | |
| 143 | |
| 144 | |
| 145 def _describe_token_type(token_type): | |
| 146 if token_type in reverse_operators: | |
| 147 return reverse_operators[token_type] | |
| 148 return { | |
| 149 TOKEN_COMMENT_BEGIN: 'begin of comment', | |
| 150 TOKEN_COMMENT_END: 'end of comment', | |
| 151 TOKEN_COMMENT: 'comment', | |
| 152 TOKEN_LINECOMMENT: 'comment', | |
| 153 TOKEN_BLOCK_BEGIN: 'begin of statement block', | |
| 154 TOKEN_BLOCK_END: 'end of statement block', | |
| 155 TOKEN_VARIABLE_BEGIN: 'begin of print statement', | |
| 156 TOKEN_VARIABLE_END: 'end of print statement', | |
| 157 TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement', | |
| 158 TOKEN_LINESTATEMENT_END: 'end of line statement', | |
| 159 TOKEN_DATA: 'template data / text', | |
| 160 TOKEN_EOF: 'end of template' | |
| 161 }.get(token_type, token_type) | |
| 162 | |
| 163 | |
| 164 def describe_token(token): | |
| 165 """Returns a description of the token.""" | |
| 166 if token.type == 'name': | |
| 167 return token.value | |
| 168 return _describe_token_type(token.type) | |
| 169 | |
| 170 | |
| 171 def describe_token_expr(expr): | |
| 172 """Like `describe_token` but for token expressions.""" | |
| 173 if ':' in expr: | |
| 174 type, value = expr.split(':', 1) | |
| 175 if type == 'name': | |
| 176 return value | |
| 177 else: | |
| 178 type = expr | |
| 179 return _describe_token_type(type) | |
| 180 | |
| 181 | |
| 182 def count_newlines(value): | |
| 183 """Count the number of newline characters in the string. This is | |
| 184 useful for extensions that filter a stream. | |
| 185 """ | |
| 186 return len(newline_re.findall(value)) | |
| 187 | |
| 188 | |
| 189 def compile_rules(environment): | |
| 190 """Compiles all the rules from the environment into a list of rules.""" | |
| 191 e = re.escape | |
| 192 rules = [ | |
| 193 (len(environment.comment_start_string), 'comment', | |
| 194 e(environment.comment_start_string)), | |
| 195 (len(environment.block_start_string), 'block', | |
| 196 e(environment.block_start_string)), | |
| 197 (len(environment.variable_start_string), 'variable', | |
| 198 e(environment.variable_start_string)) | |
| 199 ] | |
| 200 | |
| 201 if environment.line_statement_prefix is not None: | |
| 202 rules.append((len(environment.line_statement_prefix), 'linestatement', | |
| 203 r'^[ \t\v]*' + e(environment.line_statement_prefix))) | |
| 204 if environment.line_comment_prefix is not None: | |
| 205 rules.append((len(environment.line_comment_prefix), 'linecomment', | |
| 206 r'(?:^|(?<=\S))[^\S\r\n]*' + | |
| 207 e(environment.line_comment_prefix))) | |
| 208 | |
| 209 return [x[1:] for x in sorted(rules, reverse=True)] | |
| 210 | |
| 211 | |
| 212 class Failure(object): | |
| 213 """Class that raises a `TemplateSyntaxError` if called. | |
| 214 Used by the `Lexer` to specify known errors. | |
| 215 """ | |
| 216 | |
| 217 def __init__(self, message, cls=TemplateSyntaxError): | |
| 218 self.message = message | |
| 219 self.error_class = cls | |
| 220 | |
| 221 def __call__(self, lineno, filename): | |
| 222 raise self.error_class(self.message, lineno, filename) | |
| 223 | |
| 224 | |
| 225 class Token(tuple): | |
| 226 """Token class.""" | |
| 227 __slots__ = () | |
| 228 lineno, type, value = (property(itemgetter(x)) for x in range(3)) | |
| 229 | |
| 230 def __new__(cls, lineno, type, value): | |
| 231 return tuple.__new__(cls, (lineno, intern(str(type)), value)) | |
| 232 | |
| 233 def __str__(self): | |
| 234 if self.type in reverse_operators: | |
| 235 return reverse_operators[self.type] | |
| 236 elif self.type == 'name': | |
| 237 return self.value | |
| 238 return self.type | |
| 239 | |
| 240 def test(self, expr): | |
| 241 """Test a token against a token expression. This can either be a | |
| 242 token type or ``'token_type:token_value'``. This can only test | |
| 243 against string values and types. | |
| 244 """ | |
| 245 # here we do a regular string equality check as test_any is usually | |
| 246 # passed an iterable of not interned strings. | |
| 247 if self.type == expr: | |
| 248 return True | |
| 249 elif ':' in expr: | |
| 250 return expr.split(':', 1) == [self.type, self.value] | |
| 251 return False | |
| 252 | |
| 253 def test_any(self, *iterable): | |
| 254 """Test against multiple token expressions.""" | |
| 255 for expr in iterable: | |
| 256 if self.test(expr): | |
| 257 return True | |
| 258 return False | |
| 259 | |
| 260 def __repr__(self): | |
| 261 return 'Token(%r, %r, %r)' % ( | |
| 262 self.lineno, | |
| 263 self.type, | |
| 264 self.value | |
| 265 ) | |
| 266 | |
| 267 | |
| 268 @implements_iterator | |
| 269 class TokenStreamIterator(object): | |
| 270 """The iterator for tokenstreams. Iterate over the stream | |
| 271 until the eof token is reached. | |
| 272 """ | |
| 273 | |
| 274 def __init__(self, stream): | |
| 275 self.stream = stream | |
| 276 | |
| 277 def __iter__(self): | |
| 278 return self | |
| 279 | |
| 280 def __next__(self): | |
| 281 token = self.stream.current | |
| 282 if token.type is TOKEN_EOF: | |
| 283 self.stream.close() | |
| 284 raise StopIteration() | |
| 285 next(self.stream) | |
| 286 return token | |
| 287 | |
| 288 | |
| 289 @implements_iterator | |
| 290 class TokenStream(object): | |
| 291 """A token stream is an iterable that yields :class:`Token`\s. The | |
| 292 parser however does not iterate over it but calls :meth:`next` to go | |
| 293 one token ahead. The current active token is stored as :attr:`current`. | |
| 294 """ | |
| 295 | |
| 296 def __init__(self, generator, name, filename): | |
| 297 self._iter = iter(generator) | |
| 298 self._pushed = deque() | |
| 299 self.name = name | |
| 300 self.filename = filename | |
| 301 self.closed = False | |
| 302 self.current = Token(1, TOKEN_INITIAL, '') | |
| 303 next(self) | |
| 304 | |
| 305 def __iter__(self): | |
| 306 return TokenStreamIterator(self) | |
| 307 | |
| 308 def __bool__(self): | |
| 309 return bool(self._pushed) or self.current.type is not TOKEN_EOF | |
| 310 __nonzero__ = __bool__ # py2 | |
| 311 | |
| 312 eos = property(lambda x: not x, doc="Are we at the end of the stream?") | |
| 313 | |
| 314 def push(self, token): | |
| 315 """Push a token back to the stream.""" | |
| 316 self._pushed.append(token) | |
| 317 | |
| 318 def look(self): | |
| 319 """Look at the next token.""" | |
| 320 old_token = next(self) | |
| 321 result = self.current | |
| 322 self.push(result) | |
| 323 self.current = old_token | |
| 324 return result | |
| 325 | |
| 326 def skip(self, n=1): | |
| 327 """Got n tokens ahead.""" | |
| 328 for x in range(n): | |
| 329 next(self) | |
| 330 | |
| 331 def next_if(self, expr): | |
| 332 """Perform the token test and return the token if it matched. | |
| 333 Otherwise the return value is `None`. | |
| 334 """ | |
| 335 if self.current.test(expr): | |
| 336 return next(self) | |
| 337 | |
| 338 def skip_if(self, expr): | |
| 339 """Like :meth:`next_if` but only returns `True` or `False`.""" | |
| 340 return self.next_if(expr) is not None | |
| 341 | |
| 342 def __next__(self): | |
| 343 """Go one token ahead and return the old one""" | |
| 344 rv = self.current | |
| 345 if self._pushed: | |
| 346 self.current = self._pushed.popleft() | |
| 347 elif self.current.type is not TOKEN_EOF: | |
| 348 try: | |
| 349 self.current = next(self._iter) | |
| 350 except StopIteration: | |
| 351 self.close() | |
| 352 return rv | |
| 353 | |
| 354 def close(self): | |
| 355 """Close the stream.""" | |
| 356 self.current = Token(self.current.lineno, TOKEN_EOF, '') | |
| 357 self._iter = None | |
| 358 self.closed = True | |
| 359 | |
| 360 def expect(self, expr): | |
| 361 """Expect a given token type and return it. This accepts the same | |
| 362 argument as :meth:`jinja2.lexer.Token.test`. | |
| 363 """ | |
| 364 if not self.current.test(expr): | |
| 365 expr = describe_token_expr(expr) | |
| 366 if self.current.type is TOKEN_EOF: | |
| 367 raise TemplateSyntaxError('unexpected end of template, ' | |
| 368 'expected %r.' % expr, | |
| 369 self.current.lineno, | |
| 370 self.name, self.filename) | |
| 371 raise TemplateSyntaxError("expected token %r, got %r" % | |
| 372 (expr, describe_token(self.current)), | |
| 373 self.current.lineno, | |
| 374 self.name, self.filename) | |
| 375 try: | |
| 376 return self.current | |
| 377 finally: | |
| 378 next(self) | |
| 379 | |
| 380 | |
| 381 def get_lexer(environment): | |
| 382 """Return a lexer which is probably cached.""" | |
| 383 key = (environment.block_start_string, | |
| 384 environment.block_end_string, | |
| 385 environment.variable_start_string, | |
| 386 environment.variable_end_string, | |
| 387 environment.comment_start_string, | |
| 388 environment.comment_end_string, | |
| 389 environment.line_statement_prefix, | |
| 390 environment.line_comment_prefix, | |
| 391 environment.trim_blocks, | |
| 392 environment.lstrip_blocks, | |
| 393 environment.newline_sequence, | |
| 394 environment.keep_trailing_newline) | |
| 395 lexer = _lexer_cache.get(key) | |
| 396 if lexer is None: | |
| 397 lexer = Lexer(environment) | |
| 398 _lexer_cache[key] = lexer | |
| 399 return lexer | |
| 400 | |
| 401 | |
| 402 class Lexer(object): | |
| 403 """Class that implements a lexer for a given environment. Automatically | |
| 404 created by the environment class, usually you don't have to do that. | |
| 405 | |
| 406 Note that the lexer is not automatically bound to an environment. | |
| 407 Multiple environments can share the same lexer. | |
| 408 """ | |
| 409 | |
| 410 def __init__(self, environment): | |
| 411 # shortcuts | |
| 412 c = lambda x: re.compile(x, re.M | re.S) | |
| 413 e = re.escape | |
| 414 | |
| 415 # lexing rules for tags | |
| 416 tag_rules = [ | |
| 417 (whitespace_re, TOKEN_WHITESPACE, None), | |
| 418 (float_re, TOKEN_FLOAT, None), | |
| 419 (integer_re, TOKEN_INTEGER, None), | |
| 420 (name_re, TOKEN_NAME, None), | |
| 421 (string_re, TOKEN_STRING, None), | |
| 422 (operator_re, TOKEN_OPERATOR, None) | |
| 423 ] | |
| 424 | |
| 425 # assemble the root lexing rule. because "|" is ungreedy | |
| 426 # we have to sort by length so that the lexer continues working | |
| 427 # as expected when we have parsing rules like <% for block and | |
| 428 # <%= for variables. (if someone wants asp like syntax) | |
| 429 # variables are just part of the rules if variable processing | |
| 430 # is required. | |
| 431 root_tag_rules = compile_rules(environment) | |
| 432 | |
| 433 # block suffix if trimming is enabled | |
| 434 block_suffix_re = environment.trim_blocks and '\\n?' or '' | |
| 435 | |
| 436 # strip leading spaces if lstrip_blocks is enabled | |
| 437 prefix_re = {} | |
| 438 if environment.lstrip_blocks: | |
| 439 # use '{%+' to manually disable lstrip_blocks behavior | |
| 440 no_lstrip_re = e('+') | |
| 441 # detect overlap between block and variable or comment strings | |
| 442 block_diff = c(r'^%s(.*)' % e(environment.block_start_string)) | |
| 443 # make sure we don't mistake a block for a variable or a comment | |
| 444 m = block_diff.match(environment.comment_start_string) | |
| 445 no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' | |
| 446 m = block_diff.match(environment.variable_start_string) | |
| 447 no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' | |
| 448 | |
| 449 # detect overlap between comment and variable strings | |
| 450 comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string)) | |
| 451 m = comment_diff.match(environment.variable_start_string) | |
| 452 no_variable_re = m and r'(?!%s)' % e(m.group(1)) or '' | |
| 453 | |
| 454 lstrip_re = r'^[ \t]*' | |
| 455 block_prefix_re = r'%s%s(?!%s)|%s\+?' % ( | |
| 456 lstrip_re, | |
| 457 e(environment.block_start_string), | |
| 458 no_lstrip_re, | |
| 459 e(environment.block_start_string), | |
| 460 ) | |
| 461 comment_prefix_re = r'%s%s%s|%s\+?' % ( | |
| 462 lstrip_re, | |
| 463 e(environment.comment_start_string), | |
| 464 no_variable_re, | |
| 465 e(environment.comment_start_string), | |
| 466 ) | |
| 467 prefix_re['block'] = block_prefix_re | |
| 468 prefix_re['comment'] = comment_prefix_re | |
| 469 else: | |
| 470 block_prefix_re = '%s' % e(environment.block_start_string) | |
| 471 | |
| 472 self.newline_sequence = environment.newline_sequence | |
| 473 self.keep_trailing_newline = environment.keep_trailing_newline | |
| 474 | |
| 475 # global lexing rules | |
| 476 self.rules = { | |
| 477 'root': [ | |
| 478 # directives | |
| 479 (c('(.*?)(?:%s)' % '|'.join( | |
| 480 [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( | |
| 481 e(environment.block_start_string), | |
| 482 block_prefix_re, | |
| 483 e(environment.block_end_string), | |
| 484 e(environment.block_end_string) | |
| 485 )] + [ | |
| 486 r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r)) | |
| 487 for n, r in root_tag_rules | |
| 488 ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), | |
| 489 # data | |
| 490 (c('.+'), TOKEN_DATA, None) | |
| 491 ], | |
| 492 # comments | |
| 493 TOKEN_COMMENT_BEGIN: [ | |
| 494 (c(r'(.*?)((?:\-%s\s*|%s)%s)' % ( | |
| 495 e(environment.comment_end_string), | |
| 496 e(environment.comment_end_string), | |
| 497 block_suffix_re | |
| 498 )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'), | |
| 499 (c('(.)'), (Failure('Missing end of comment tag'),), None) | |
| 500 ], | |
| 501 # blocks | |
| 502 TOKEN_BLOCK_BEGIN: [ | |
| 503 (c('(?:\-%s\s*|%s)%s' % ( | |
| 504 e(environment.block_end_string), | |
| 505 e(environment.block_end_string), | |
| 506 block_suffix_re | |
| 507 )), TOKEN_BLOCK_END, '#pop'), | |
| 508 ] + tag_rules, | |
| 509 # variables | |
| 510 TOKEN_VARIABLE_BEGIN: [ | |
| 511 (c('\-%s\s*|%s' % ( | |
| 512 e(environment.variable_end_string), | |
| 513 e(environment.variable_end_string) | |
| 514 )), TOKEN_VARIABLE_END, '#pop') | |
| 515 ] + tag_rules, | |
| 516 # raw block | |
| 517 TOKEN_RAW_BEGIN: [ | |
| 518 (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( | |
| 519 e(environment.block_start_string), | |
| 520 block_prefix_re, | |
| 521 e(environment.block_end_string), | |
| 522 e(environment.block_end_string), | |
| 523 block_suffix_re | |
| 524 )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'), | |
| 525 (c('(.)'), (Failure('Missing end of raw directive'),), None) | |
| 526 ], | |
| 527 # line statements | |
| 528 TOKEN_LINESTATEMENT_BEGIN: [ | |
| 529 (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop') | |
| 530 ] + tag_rules, | |
| 531 # line comments | |
| 532 TOKEN_LINECOMMENT_BEGIN: [ | |
| 533 (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT, | |
| 534 TOKEN_LINECOMMENT_END), '#pop') | |
| 535 ] | |
| 536 } | |
| 537 | |
| 538 def _normalize_newlines(self, value): | |
| 539 """Called for strings and template data to normalize it to unicode.""" | |
| 540 return newline_re.sub(self.newline_sequence, value) | |
| 541 | |
| 542 def tokenize(self, source, name=None, filename=None, state=None): | |
| 543 """Calls tokeniter + tokenize and wraps it in a token stream. | |
| 544 """ | |
| 545 stream = self.tokeniter(source, name, filename, state) | |
| 546 return TokenStream(self.wrap(stream, name, filename), name, filename) | |
| 547 | |
| 548 def wrap(self, stream, name=None, filename=None): | |
| 549 """This is called with the stream as returned by `tokenize` and wraps | |
| 550 every token in a :class:`Token` and converts the value. | |
| 551 """ | |
| 552 for lineno, token, value in stream: | |
| 553 if token in ignored_tokens: | |
| 554 continue | |
| 555 elif token == 'linestatement_begin': | |
| 556 token = 'block_begin' | |
| 557 elif token == 'linestatement_end': | |
| 558 token = 'block_end' | |
| 559 # we are not interested in those tokens in the parser | |
| 560 elif token in ('raw_begin', 'raw_end'): | |
| 561 continue | |
| 562 elif token == 'data': | |
| 563 value = self._normalize_newlines(value) | |
| 564 elif token == 'keyword': | |
| 565 token = value | |
| 566 elif token == 'name': | |
| 567 value = str(value) | |
| 568 elif token == 'string': | |
| 569 # try to unescape string | |
| 570 try: | |
| 571 value = self._normalize_newlines(value[1:-1]) \ | |
| 572 .encode('ascii', 'backslashreplace') \ | |
| 573 .decode('unicode-escape') | |
| 574 except Exception as e: | |
| 575 msg = str(e).split(':')[-1].strip() | |
| 576 raise TemplateSyntaxError(msg, lineno, name, filename) | |
| 577 # if we can express it as bytestring (ascii only) | |
| 578 # we do that for support of semi broken APIs | |
| 579 # as datetime.datetime.strftime. On python 3 this | |
| 580 # call becomes a noop thanks to 2to3 | |
| 581 try: | |
| 582 value = str(value) | |
| 583 except UnicodeError: | |
| 584 pass | |
| 585 elif token == 'integer': | |
| 586 value = int(value) | |
| 587 elif token == 'float': | |
| 588 value = float(value) | |
| 589 elif token == 'operator': | |
| 590 token = operators[value] | |
| 591 yield Token(lineno, token, value) | |
| 592 | |
| 593 def tokeniter(self, source, name, filename=None, state=None): | |
| 594 """This method tokenizes the text and returns the tokens in a | |
| 595 generator. Use this method if you just want to tokenize a template. | |
| 596 """ | |
| 597 source = text_type(source) | |
| 598 lines = source.splitlines() | |
| 599 if self.keep_trailing_newline and source: | |
| 600 for newline in ('\r\n', '\r', '\n'): | |
| 601 if source.endswith(newline): | |
| 602 lines.append('') | |
| 603 break | |
| 604 source = '\n'.join(lines) | |
| 605 pos = 0 | |
| 606 lineno = 1 | |
| 607 stack = ['root'] | |
| 608 if state is not None and state != 'root': | |
| 609 assert state in ('variable', 'block'), 'invalid state' | |
| 610 stack.append(state + '_begin') | |
| 611 else: | |
| 612 state = 'root' | |
| 613 statetokens = self.rules[stack[-1]] | |
| 614 source_length = len(source) | |
| 615 | |
| 616 balancing_stack = [] | |
| 617 | |
| 618 while 1: | |
| 619 # tokenizer loop | |
| 620 for regex, tokens, new_state in statetokens: | |
| 621 m = regex.match(source, pos) | |
| 622 # if no match we try again with the next rule | |
| 623 if m is None: | |
| 624 continue | |
| 625 | |
| 626 # we only match blocks and variables if braces / parentheses | |
| 627 # are balanced. continue parsing with the lower rule which | |
| 628 # is the operator rule. do this only if the end tags look | |
| 629 # like operators | |
| 630 if balancing_stack and \ | |
| 631 tokens in ('variable_end', 'block_end', | |
| 632 'linestatement_end'): | |
| 633 continue | |
| 634 | |
| 635 # tuples support more options | |
| 636 if isinstance(tokens, tuple): | |
| 637 for idx, token in enumerate(tokens): | |
| 638 # failure group | |
| 639 if token.__class__ is Failure: | |
| 640 raise token(lineno, filename) | |
| 641 # bygroup is a bit more complex, in that case we | |
| 642 # yield for the current token the first named | |
| 643 # group that matched | |
| 644 elif token == '#bygroup': | |
| 645 for key, value in iteritems(m.groupdict()): | |
| 646 if value is not None: | |
| 647 yield lineno, key, value | |
| 648 lineno += value.count('\n') | |
| 649 break | |
| 650 else: | |
| 651 raise RuntimeError('%r wanted to resolve ' | |
| 652 'the token dynamically' | |
| 653 ' but no group matched' | |
| 654 % regex) | |
| 655 # normal group | |
| 656 else: | |
| 657 data = m.group(idx + 1) | |
| 658 if data or token not in ignore_if_empty: | |
| 659 yield lineno, token, data | |
| 660 lineno += data.count('\n') | |
| 661 | |
| 662 # strings as token just are yielded as it. | |
| 663 else: | |
| 664 data = m.group() | |
| 665 # update brace/parentheses balance | |
| 666 if tokens == 'operator': | |
| 667 if data == '{': | |
| 668 balancing_stack.append('}') | |
| 669 elif data == '(': | |
| 670 balancing_stack.append(')') | |
| 671 elif data == '[': | |
| 672 balancing_stack.append(']') | |
| 673 elif data in ('}', ')', ']'): | |
| 674 if not balancing_stack: | |
| 675 raise TemplateSyntaxError('unexpected \'%s\'' % | |
| 676 data, lineno, name, | |
| 677 filename) | |
| 678 expected_op = balancing_stack.pop() | |
| 679 if expected_op != data: | |
| 680 raise TemplateSyntaxError('unexpected \'%s\', ' | |
| 681 'expected \'%s\'' % | |
| 682 (data, expected_op), | |
| 683 lineno, name, | |
| 684 filename) | |
| 685 # yield items | |
| 686 if data or tokens not in ignore_if_empty: | |
| 687 yield lineno, tokens, data | |
| 688 lineno += data.count('\n') | |
| 689 | |
| 690 # fetch new position into new variable so that we can check | |
| 691 # if there is a internal parsing error which would result | |
| 692 # in an infinite loop | |
| 693 pos2 = m.end() | |
| 694 | |
| 695 # handle state changes | |
| 696 if new_state is not None: | |
| 697 # remove the uppermost state | |
| 698 if new_state == '#pop': | |
| 699 stack.pop() | |
| 700 # resolve the new state by group checking | |
| 701 elif new_state == '#bygroup': | |
| 702 for key, value in iteritems(m.groupdict()): | |
| 703 if value is not None: | |
| 704 stack.append(key) | |
| 705 break | |
| 706 else: | |
| 707 raise RuntimeError('%r wanted to resolve the ' | |
| 708 'new state dynamically but' | |
| 709 ' no group matched' % | |
| 710 regex) | |
| 711 # direct state name given | |
| 712 else: | |
| 713 stack.append(new_state) | |
| 714 statetokens = self.rules[stack[-1]] | |
| 715 # we are still at the same position and no stack change. | |
| 716 # this means a loop without break condition, avoid that and | |
| 717 # raise error | |
| 718 elif pos2 == pos: | |
| 719 raise RuntimeError('%r yielded empty string without ' | |
| 720 'stack change' % regex) | |
| 721 # publish new function and start again | |
| 722 pos = pos2 | |
| 723 break | |
| 724 # if loop terminated without break we haven't found a single match | |
| 725 # either we are at the end of the file or we have a problem | |
| 726 else: | |
| 727 # end of text | |
| 728 if pos >= source_length: | |
| 729 return | |
| 730 # something went wrong | |
| 731 raise TemplateSyntaxError('unexpected char %r at %d' % | |
| 732 (source[pos], pos), lineno, | |
| 733 name, filename) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 814543006:
Move //mojo/{public, edk} underneath //third_party (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master