OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import imp | 5 import imp |
6 import os.path | 6 import os.path |
7 import sys | 7 import sys |
8 | 8 |
9 # Disable lint check for finding modules: | 9 # Disable lint check for finding modules: |
10 # pylint: disable=F0401 | 10 # pylint: disable=F0401 |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
72 | 72 |
73 ## | 73 ## |
74 ## All the tokens recognized by the lexer | 74 ## All the tokens recognized by the lexer |
75 ## | 75 ## |
76 tokens = keywords + ( | 76 tokens = keywords + ( |
77 # Identifiers | 77 # Identifiers |
78 'NAME', | 78 'NAME', |
79 | 79 |
80 # Constants | 80 # Constants |
81 'ORDINAL', | 81 'ORDINAL', |
82 'INT_CONST_DEC', 'INT_CONST_OCT', 'INT_CONST_HEX', | 82 'INT_CONST_DEC', 'INT_CONST_HEX', |
83 'FLOAT_CONST', | 83 'FLOAT_CONST', |
84 'CHAR_CONST', | 84 'CHAR_CONST', |
85 | 85 |
86 # String literals | 86 # String literals |
87 'STRING_LITERAL', | 87 'STRING_LITERAL', |
88 | 88 |
89 # Operators | 89 # Operators |
90 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', | 90 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', |
91 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', | 91 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', |
92 | 92 |
(...skipping 17 matching lines...) Expand all Loading... |
110 ## | 110 ## |
111 | 111 |
112 # valid C identifiers (K&R2: A.2.3) | 112 # valid C identifiers (K&R2: A.2.3) |
113 identifier = r'[a-zA-Z_][0-9a-zA-Z_]*' | 113 identifier = r'[a-zA-Z_][0-9a-zA-Z_]*' |
114 | 114 |
115 hex_prefix = '0[xX]' | 115 hex_prefix = '0[xX]' |
116 hex_digits = '[0-9a-fA-F]+' | 116 hex_digits = '[0-9a-fA-F]+' |
117 | 117 |
118 # integer constants (K&R2: A.2.5.1) | 118 # integer constants (K&R2: A.2.5.1) |
119 decimal_constant = '0|([1-9][0-9]*)' | 119 decimal_constant = '0|([1-9][0-9]*)' |
120 octal_constant = '0[0-7]+' | |
121 hex_constant = hex_prefix+hex_digits | 120 hex_constant = hex_prefix+hex_digits |
122 | 121 # Don't allow octal constants (even invalid octal). |
123 bad_octal_constant = '0[0-7]*[89]' | 122 octal_constant_disallowed = '0[0-9]+' |
124 | 123 |
125 # character constants (K&R2: A.2.5.2) | 124 # character constants (K&R2: A.2.5.2) |
126 # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line | 125 # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line |
127 # directives with Windows paths as filenames (..\..\dir\file) | 126 # directives with Windows paths as filenames (..\..\dir\file) |
128 # For the same reason, decimal_escape allows all digit sequences. We want to | 127 # For the same reason, decimal_escape allows all digit sequences. We want to |
129 # parse all correct code, even if it means to sometimes parse incorrect | 128 # parse all correct code, even if it means to sometimes parse incorrect |
130 # code. | 129 # code. |
131 # | 130 # |
132 simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" | 131 simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" |
133 decimal_escape = r"""(\d+)""" | 132 decimal_escape = r"""(\d+)""" |
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
212 # and this is bad) | 211 # and this is bad) |
213 # | 212 # |
214 @TOKEN(floating_constant) | 213 @TOKEN(floating_constant) |
215 def t_FLOAT_CONST(self, t): | 214 def t_FLOAT_CONST(self, t): |
216 return t | 215 return t |
217 | 216 |
218 @TOKEN(hex_constant) | 217 @TOKEN(hex_constant) |
219 def t_INT_CONST_HEX(self, t): | 218 def t_INT_CONST_HEX(self, t): |
220 return t | 219 return t |
221 | 220 |
222 @TOKEN(bad_octal_constant) | 221 @TOKEN(octal_constant_disallowed) |
223 def t_BAD_CONST_OCT(self, t): | 222 def t_OCTAL_CONSTANT_DISALLOWED(self, t): |
224 msg = "Invalid octal constant" | 223 msg = "Octal values not allowed" |
225 self._error(msg, t) | 224 self._error(msg, t) |
226 | 225 |
227 @TOKEN(octal_constant) | |
228 def t_INT_CONST_OCT(self, t): | |
229 return t | |
230 | |
231 @TOKEN(decimal_constant) | 226 @TOKEN(decimal_constant) |
232 def t_INT_CONST_DEC(self, t): | 227 def t_INT_CONST_DEC(self, t): |
233 return t | 228 return t |
234 | 229 |
235 # Must come before bad_char_const, to prevent it from | 230 # Must come before bad_char_const, to prevent it from |
236 # catching valid char constants as invalid | 231 # catching valid char constants as invalid |
237 # | 232 # |
238 @TOKEN(char_const) | 233 @TOKEN(char_const) |
239 def t_CHAR_CONST(self, t): | 234 def t_CHAR_CONST(self, t): |
240 return t | 235 return t |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
277 return t | 272 return t |
278 | 273 |
279 # Ignore C and C++ style comments | 274 # Ignore C and C++ style comments |
280 def t_COMMENT(self, t): | 275 def t_COMMENT(self, t): |
281 r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' | 276 r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' |
282 t.lexer.lineno += t.value.count("\n") | 277 t.lexer.lineno += t.value.count("\n") |
283 | 278 |
284 def t_error(self, t): | 279 def t_error(self, t): |
285 msg = "Illegal character %s" % repr(t.value[0]) | 280 msg = "Illegal character %s" % repr(t.value[0]) |
286 self._error(msg, t) | 281 self._error(msg, t) |
OLD | NEW |