| OLD | NEW |
| 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "tools/gn/tokenizer.h" | 5 #include "tools/gn/tokenizer.h" |
| 6 | 6 |
| 7 #include "base/logging.h" | 7 #include "base/logging.h" |
| 8 #include "base/strings/string_util.h" | 8 #include "base/strings/string_util.h" |
| 9 #include "tools/gn/input_file.h" | 9 #include "tools/gn/input_file.h" |
| 10 | 10 |
| (...skipping 170 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 181 | 181 |
| 182 | 182 |
| 183 void Tokenizer::AdvanceToNextToken() { | 183 void Tokenizer::AdvanceToNextToken() { |
| 184 while (!at_end() && IsCurrentWhitespace()) | 184 while (!at_end() && IsCurrentWhitespace()) |
| 185 Advance(); | 185 Advance(); |
| 186 } | 186 } |
| 187 | 187 |
| 188 Token::Type Tokenizer::ClassifyCurrent() const { | 188 Token::Type Tokenizer::ClassifyCurrent() const { |
| 189 DCHECK(!at_end()); | 189 DCHECK(!at_end()); |
| 190 char next_char = cur_char(); | 190 char next_char = cur_char(); |
| 191 if (IsAsciiDigit(next_char)) | 191 if (base::IsAsciiDigit(next_char)) |
| 192 return Token::INTEGER; | 192 return Token::INTEGER; |
| 193 if (next_char == '"') | 193 if (next_char == '"') |
| 194 return Token::STRING; | 194 return Token::STRING; |
| 195 | 195 |
| 196 // Note: '-' handled specially below. | 196 // Note: '-' handled specially below. |
| 197 if (next_char != '-' && CouldBeOperator(next_char)) | 197 if (next_char != '-' && CouldBeOperator(next_char)) |
| 198 return Token::UNCLASSIFIED_OPERATOR; | 198 return Token::UNCLASSIFIED_OPERATOR; |
| 199 | 199 |
| 200 if (IsIdentifierFirstChar(next_char)) | 200 if (IsIdentifierFirstChar(next_char)) |
| 201 return Token::IDENTIFIER; | 201 return Token::IDENTIFIER; |
| (...skipping 19 matching lines...) Expand all Loading... |
| 221 if (next_char == '#') | 221 if (next_char == '#') |
| 222 return Token::UNCLASSIFIED_COMMENT; | 222 return Token::UNCLASSIFIED_COMMENT; |
| 223 | 223 |
| 224 // For the case of '-' differentiate between a negative number and anything | 224 // For the case of '-' differentiate between a negative number and anything |
| 225 // else. | 225 // else. |
| 226 if (next_char == '-') { | 226 if (next_char == '-') { |
| 227 if (!CanIncrement()) | 227 if (!CanIncrement()) |
| 228 return Token::UNCLASSIFIED_OPERATOR; // Just the minus before end of | 228 return Token::UNCLASSIFIED_OPERATOR; // Just the minus before end of |
| 229 // file. | 229 // file. |
| 230 char following_char = input_[cur_ + 1]; | 230 char following_char = input_[cur_ + 1]; |
| 231 if (IsAsciiDigit(following_char)) | 231 if (base::IsAsciiDigit(following_char)) |
| 232 return Token::INTEGER; | 232 return Token::INTEGER; |
| 233 return Token::UNCLASSIFIED_OPERATOR; | 233 return Token::UNCLASSIFIED_OPERATOR; |
| 234 } | 234 } |
| 235 | 235 |
| 236 return Token::INVALID; | 236 return Token::INVALID; |
| 237 } | 237 } |
| 238 | 238 |
| 239 void Tokenizer::AdvanceToEndOfToken(const Location& location, | 239 void Tokenizer::AdvanceToEndOfToken(const Location& location, |
| 240 Token::Type type) { | 240 Token::Type type) { |
| 241 switch (type) { | 241 switch (type) { |
| 242 case Token::INTEGER: | 242 case Token::INTEGER: |
| 243 do { | 243 do { |
| 244 Advance(); | 244 Advance(); |
| 245 } while (!at_end() && IsAsciiDigit(cur_char())); | 245 } while (!at_end() && base::IsAsciiDigit(cur_char())); |
| 246 if (!at_end()) { | 246 if (!at_end()) { |
| 247 // Require the char after a number to be some kind of space, scope, | 247 // Require the char after a number to be some kind of space, scope, |
| 248 // or operator. | 248 // or operator. |
| 249 char c = cur_char(); | 249 char c = cur_char(); |
| 250 if (!IsCurrentWhitespace() && !CouldBeOperator(c) && | 250 if (!IsCurrentWhitespace() && !CouldBeOperator(c) && |
| 251 !IsScoperChar(c) && c != ',') { | 251 !IsScoperChar(c) && c != ',') { |
| 252 *err_ = Err(GetCurrentLocation(), | 252 *err_ = Err(GetCurrentLocation(), |
| 253 "This is not a valid number.", | 253 "This is not a valid number.", |
| 254 "Learn to count."); | 254 "Learn to count."); |
| 255 // Highlight the number. | 255 // Highlight the number. |
| (...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 389 // Different types of comments. | 389 // Different types of comments. |
| 390 help = "Comments should start with # instead"; | 390 help = "Comments should start with # instead"; |
| 391 } else if (cur_char() == '\'') { | 391 } else if (cur_char() == '\'') { |
| 392 help = "Strings are delimited by \" characters, not apostrophes."; | 392 help = "Strings are delimited by \" characters, not apostrophes."; |
| 393 } else { | 393 } else { |
| 394 help = "I have no idea what this is."; | 394 help = "I have no idea what this is."; |
| 395 } | 395 } |
| 396 | 396 |
| 397 return Err(location, "Invalid token.", help); | 397 return Err(location, "Invalid token.", help); |
| 398 } | 398 } |
| OLD | NEW |