| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import imp | 5 import imp |
| 6 import os.path | 6 import os.path |
| 7 import sys | 7 import sys |
| 8 | 8 |
| 9 # Disable lint check for finding modules: | 9 # Disable lint check for finding modules: |
| 10 # pylint: disable=F0401 | 10 # pylint: disable=F0401 |
| (...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 72 | 72 |
| 73 ## | 73 ## |
| 74 ## All the tokens recognized by the lexer | 74 ## All the tokens recognized by the lexer |
| 75 ## | 75 ## |
| 76 tokens = keywords + ( | 76 tokens = keywords + ( |
| 77 # Identifiers | 77 # Identifiers |
| 78 'NAME', | 78 'NAME', |
| 79 | 79 |
| 80 # Constants | 80 # Constants |
| 81 'ORDINAL', | 81 'ORDINAL', |
| 82 'INT_CONST_DEC', 'INT_CONST_OCT', 'INT_CONST_HEX', | 82 'INT_CONST_DEC', 'INT_CONST_HEX', |
| 83 'FLOAT_CONST', | 83 'FLOAT_CONST', |
| 84 'CHAR_CONST', | 84 'CHAR_CONST', |
| 85 | 85 |
| 86 # String literals | 86 # String literals |
| 87 'STRING_LITERAL', | 87 'STRING_LITERAL', |
| 88 | 88 |
| 89 # Operators | 89 # Operators |
| 90 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', | 90 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', |
| 91 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', | 91 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', |
| 92 | 92 |
| (...skipping 17 matching lines...) Expand all Loading... |
| 110 ## | 110 ## |
| 111 | 111 |
| 112 # valid C identifiers (K&R2: A.2.3) | 112 # valid C identifiers (K&R2: A.2.3) |
| 113 identifier = r'[a-zA-Z_][0-9a-zA-Z_]*' | 113 identifier = r'[a-zA-Z_][0-9a-zA-Z_]*' |
| 114 | 114 |
| 115 hex_prefix = '0[xX]' | 115 hex_prefix = '0[xX]' |
| 116 hex_digits = '[0-9a-fA-F]+' | 116 hex_digits = '[0-9a-fA-F]+' |
| 117 | 117 |
| 118 # integer constants (K&R2: A.2.5.1) | 118 # integer constants (K&R2: A.2.5.1) |
| 119 decimal_constant = '0|([1-9][0-9]*)' | 119 decimal_constant = '0|([1-9][0-9]*)' |
| 120 octal_constant = '0[0-7]+' | |
| 121 hex_constant = hex_prefix+hex_digits | 120 hex_constant = hex_prefix+hex_digits |
| 122 | 121 # Don't allow octal constants (even invalid octal). |
| 123 bad_octal_constant = '0[0-7]*[89]' | 122 octal_constant_disallowed = '0[0-9]+' |
| 124 | 123 |
| 125 # character constants (K&R2: A.2.5.2) | 124 # character constants (K&R2: A.2.5.2) |
| 126 # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line | 125 # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line |
| 127 # directives with Windows paths as filenames (..\..\dir\file) | 126 # directives with Windows paths as filenames (..\..\dir\file) |
| 128 # For the same reason, decimal_escape allows all digit sequences. We want to | 127 # For the same reason, decimal_escape allows all digit sequences. We want to |
| 129 # parse all correct code, even if it means to sometimes parse incorrect | 128 # parse all correct code, even if it means to sometimes parse incorrect |
| 130 # code. | 129 # code. |
| 131 # | 130 # |
| 132 simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" | 131 simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" |
| 133 decimal_escape = r"""(\d+)""" | 132 decimal_escape = r"""(\d+)""" |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 212 # and this is bad) | 211 # and this is bad) |
| 213 # | 212 # |
| 214 @TOKEN(floating_constant) | 213 @TOKEN(floating_constant) |
| 215 def t_FLOAT_CONST(self, t): | 214 def t_FLOAT_CONST(self, t): |
| 216 return t | 215 return t |
| 217 | 216 |
| 218 @TOKEN(hex_constant) | 217 @TOKEN(hex_constant) |
| 219 def t_INT_CONST_HEX(self, t): | 218 def t_INT_CONST_HEX(self, t): |
| 220 return t | 219 return t |
| 221 | 220 |
| 222 @TOKEN(bad_octal_constant) | 221 @TOKEN(octal_constant_disallowed) |
| 223 def t_BAD_CONST_OCT(self, t): | 222 def t_OCTAL_CONSTANT_DISALLOWED(self, t): |
| 224 msg = "Invalid octal constant" | 223 msg = "Octal values not allowed" |
| 225 self._error(msg, t) | 224 self._error(msg, t) |
| 226 | 225 |
| 227 @TOKEN(octal_constant) | |
| 228 def t_INT_CONST_OCT(self, t): | |
| 229 return t | |
| 230 | |
| 231 @TOKEN(decimal_constant) | 226 @TOKEN(decimal_constant) |
| 232 def t_INT_CONST_DEC(self, t): | 227 def t_INT_CONST_DEC(self, t): |
| 233 return t | 228 return t |
| 234 | 229 |
| 235 # Must come before bad_char_const, to prevent it from | 230 # Must come before bad_char_const, to prevent it from |
| 236 # catching valid char constants as invalid | 231 # catching valid char constants as invalid |
| 237 # | 232 # |
| 238 @TOKEN(char_const) | 233 @TOKEN(char_const) |
| 239 def t_CHAR_CONST(self, t): | 234 def t_CHAR_CONST(self, t): |
| 240 return t | 235 return t |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 277 return t | 272 return t |
| 278 | 273 |
| 279 # Ignore C and C++ style comments | 274 # Ignore C and C++ style comments |
| 280 def t_COMMENT(self, t): | 275 def t_COMMENT(self, t): |
| 281 r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' | 276 r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' |
| 282 t.lexer.lineno += t.value.count("\n") | 277 t.lexer.lineno += t.value.count("\n") |
| 283 | 278 |
| 284 def t_error(self, t): | 279 def t_error(self, t): |
| 285 msg = "Illegal character %s" % repr(t.value[0]) | 280 msg = "Illegal character %s" % repr(t.value[0]) |
| 286 self._error(msg, t) | 281 self._error(msg, t) |
| OLD | NEW |