OLD | NEW |
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "tools/gn/tokenizer.h" | 5 #include "tools/gn/tokenizer.h" |
6 | 6 |
7 #include "base/logging.h" | 7 #include "base/logging.h" |
8 #include "base/strings/string_util.h" | 8 #include "base/strings/string_util.h" |
9 #include "tools/gn/input_file.h" | 9 #include "tools/gn/input_file.h" |
10 | 10 |
(...skipping 170 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
181 | 181 |
182 | 182 |
183 void Tokenizer::AdvanceToNextToken() { | 183 void Tokenizer::AdvanceToNextToken() { |
184 while (!at_end() && IsCurrentWhitespace()) | 184 while (!at_end() && IsCurrentWhitespace()) |
185 Advance(); | 185 Advance(); |
186 } | 186 } |
187 | 187 |
188 Token::Type Tokenizer::ClassifyCurrent() const { | 188 Token::Type Tokenizer::ClassifyCurrent() const { |
189 DCHECK(!at_end()); | 189 DCHECK(!at_end()); |
190 char next_char = cur_char(); | 190 char next_char = cur_char(); |
191 if (IsAsciiDigit(next_char)) | 191 if (base::IsAsciiDigit(next_char)) |
192 return Token::INTEGER; | 192 return Token::INTEGER; |
193 if (next_char == '"') | 193 if (next_char == '"') |
194 return Token::STRING; | 194 return Token::STRING; |
195 | 195 |
196 // Note: '-' handled specially below. | 196 // Note: '-' handled specially below. |
197 if (next_char != '-' && CouldBeOperator(next_char)) | 197 if (next_char != '-' && CouldBeOperator(next_char)) |
198 return Token::UNCLASSIFIED_OPERATOR; | 198 return Token::UNCLASSIFIED_OPERATOR; |
199 | 199 |
200 if (IsIdentifierFirstChar(next_char)) | 200 if (IsIdentifierFirstChar(next_char)) |
201 return Token::IDENTIFIER; | 201 return Token::IDENTIFIER; |
(...skipping 19 matching lines...) Expand all Loading... |
221 if (next_char == '#') | 221 if (next_char == '#') |
222 return Token::UNCLASSIFIED_COMMENT; | 222 return Token::UNCLASSIFIED_COMMENT; |
223 | 223 |
224 // For the case of '-' differentiate between a negative number and anything | 224 // For the case of '-' differentiate between a negative number and anything |
225 // else. | 225 // else. |
226 if (next_char == '-') { | 226 if (next_char == '-') { |
227 if (!CanIncrement()) | 227 if (!CanIncrement()) |
228 return Token::UNCLASSIFIED_OPERATOR; // Just the minus before end of | 228 return Token::UNCLASSIFIED_OPERATOR; // Just the minus before end of |
229 // file. | 229 // file. |
230 char following_char = input_[cur_ + 1]; | 230 char following_char = input_[cur_ + 1]; |
231 if (IsAsciiDigit(following_char)) | 231 if (base::IsAsciiDigit(following_char)) |
232 return Token::INTEGER; | 232 return Token::INTEGER; |
233 return Token::UNCLASSIFIED_OPERATOR; | 233 return Token::UNCLASSIFIED_OPERATOR; |
234 } | 234 } |
235 | 235 |
236 return Token::INVALID; | 236 return Token::INVALID; |
237 } | 237 } |
238 | 238 |
239 void Tokenizer::AdvanceToEndOfToken(const Location& location, | 239 void Tokenizer::AdvanceToEndOfToken(const Location& location, |
240 Token::Type type) { | 240 Token::Type type) { |
241 switch (type) { | 241 switch (type) { |
242 case Token::INTEGER: | 242 case Token::INTEGER: |
243 do { | 243 do { |
244 Advance(); | 244 Advance(); |
245 } while (!at_end() && IsAsciiDigit(cur_char())); | 245 } while (!at_end() && base::IsAsciiDigit(cur_char())); |
246 if (!at_end()) { | 246 if (!at_end()) { |
247 // Require the char after a number to be some kind of space, scope, | 247 // Require the char after a number to be some kind of space, scope, |
248 // or operator. | 248 // or operator. |
249 char c = cur_char(); | 249 char c = cur_char(); |
250 if (!IsCurrentWhitespace() && !CouldBeOperator(c) && | 250 if (!IsCurrentWhitespace() && !CouldBeOperator(c) && |
251 !IsScoperChar(c) && c != ',') { | 251 !IsScoperChar(c) && c != ',') { |
252 *err_ = Err(GetCurrentLocation(), | 252 *err_ = Err(GetCurrentLocation(), |
253 "This is not a valid number.", | 253 "This is not a valid number.", |
254 "Learn to count."); | 254 "Learn to count."); |
255 // Highlight the number. | 255 // Highlight the number. |
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
389 // Different types of comments. | 389 // Different types of comments. |
390 help = "Comments should start with # instead"; | 390 help = "Comments should start with # instead"; |
391 } else if (cur_char() == '\'') { | 391 } else if (cur_char() == '\'') { |
392 help = "Strings are delimited by \" characters, not apostrophes."; | 392 help = "Strings are delimited by \" characters, not apostrophes."; |
393 } else { | 393 } else { |
394 help = "I have no idea what this is."; | 394 help = "I have no idea what this is."; |
395 } | 395 } |
396 | 396 |
397 return Err(location, "Invalid token.", help); | 397 return Err(location, "Invalid token.", help); |
398 } | 398 } |
OLD | NEW |