| OLD | NEW |
| 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "tools/gn/tokenizer.h" | 5 #include "tools/gn/tokenizer.h" |
| 6 | 6 |
| 7 #include "base/logging.h" | 7 #include "base/logging.h" |
| 8 #include "base/strings/string_util.h" | 8 #include "base/strings/string_util.h" |
| 9 #include "tools/gn/input_file.h" | 9 #include "tools/gn/input_file.h" |
| 10 | 10 |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 67 } | 67 } |
| 68 | 68 |
| 69 } // namespace | 69 } // namespace |
| 70 | 70 |
| 71 Tokenizer::Tokenizer(const InputFile* input_file, Err* err) | 71 Tokenizer::Tokenizer(const InputFile* input_file, Err* err) |
| 72 : input_file_(input_file), | 72 : input_file_(input_file), |
| 73 input_(input_file->contents()), | 73 input_(input_file->contents()), |
| 74 err_(err), | 74 err_(err), |
| 75 cur_(0), | 75 cur_(0), |
| 76 line_number_(1), | 76 line_number_(1), |
| 77 char_in_line_(1) { | 77 column_number_(1) { |
| 78 } | 78 } |
| 79 | 79 |
| 80 Tokenizer::~Tokenizer() { | 80 Tokenizer::~Tokenizer() { |
| 81 } | 81 } |
| 82 | 82 |
| 83 // static | 83 // static |
| 84 std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) { | 84 std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) { |
| 85 Tokenizer t(input_file, err); | 85 Tokenizer t(input_file, err); |
| 86 return t.Run(); | 86 return t.Run(); |
| 87 } | 87 } |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 119 type = Token::TRUE_TOKEN; | 119 type = Token::TRUE_TOKEN; |
| 120 else if (token_value == "false") | 120 else if (token_value == "false") |
| 121 type = Token::FALSE_TOKEN; | 121 type = Token::FALSE_TOKEN; |
| 122 } else if (type == Token::UNCLASSIFIED_COMMENT) { | 122 } else if (type == Token::UNCLASSIFIED_COMMENT) { |
| 123 if (AtStartOfLine(token_begin) && | 123 if (AtStartOfLine(token_begin) && |
| 124 // If it's a standalone comment, but is a continuation of a comment on | 124 // If it's a standalone comment, but is a continuation of a comment on |
| 125 // a previous line, then instead make it a continued suffix comment. | 125 // a previous line, then instead make it a continued suffix comment. |
| 126 (tokens_.empty() || tokens_.back().type() != Token::SUFFIX_COMMENT || | 126 (tokens_.empty() || tokens_.back().type() != Token::SUFFIX_COMMENT || |
| 127 tokens_.back().location().line_number() + 1 != | 127 tokens_.back().location().line_number() + 1 != |
| 128 location.line_number() || | 128 location.line_number() || |
| 129 tokens_.back().location().char_offset() != location.char_offset())) { | 129 tokens_.back().location().column_number() != |
| 130 location.column_number())) { |
| 130 type = Token::LINE_COMMENT; | 131 type = Token::LINE_COMMENT; |
| 131 if (!at_end()) // Could be EOF. | 132 if (!at_end()) // Could be EOF. |
| 132 Advance(); // The current \n. | 133 Advance(); // The current \n. |
| 133 // If this comment is separated from the next syntax element, then we | 134 // If this comment is separated from the next syntax element, then we |
| 134 // want to tag it as a block comment. This will become a standalone | 135 // want to tag it as a block comment. This will become a standalone |
| 135 // statement at the parser level to keep this comment separate, rather | 136 // statement at the parser level to keep this comment separate, rather |
| 136 // than attached to the subsequent statement. | 137 // than attached to the subsequent statement. |
| 137 while (!at_end() && IsCurrentWhitespace()) { | 138 while (!at_end() && IsCurrentWhitespace()) { |
| 138 if (IsCurrentNewline()) { | 139 if (IsCurrentNewline()) { |
| 139 type = Token::BLOCK_COMMENT; | 140 type = Token::BLOCK_COMMENT; |
| (...skipping 227 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 367 } | 368 } |
| 368 | 369 |
| 369 bool Tokenizer::IsCurrentNewline() const { | 370 bool Tokenizer::IsCurrentNewline() const { |
| 370 return IsNewline(input_, cur_); | 371 return IsNewline(input_, cur_); |
| 371 } | 372 } |
| 372 | 373 |
| 373 void Tokenizer::Advance() { | 374 void Tokenizer::Advance() { |
| 374 DCHECK(cur_ < input_.size()); | 375 DCHECK(cur_ < input_.size()); |
| 375 if (IsCurrentNewline()) { | 376 if (IsCurrentNewline()) { |
| 376 line_number_++; | 377 line_number_++; |
| 377 char_in_line_ = 1; | 378 column_number_ = 1; |
| 378 } else { | 379 } else { |
| 379 char_in_line_++; | 380 column_number_++; |
| 380 } | 381 } |
| 381 cur_++; | 382 cur_++; |
| 382 } | 383 } |
| 383 | 384 |
| 384 Location Tokenizer::GetCurrentLocation() const { | 385 Location Tokenizer::GetCurrentLocation() const { |
| 385 return Location( | 386 return Location( |
| 386 input_file_, line_number_, char_in_line_, static_cast<int>(cur_)); | 387 input_file_, line_number_, column_number_, static_cast<int>(cur_)); |
| 387 } | 388 } |
| 388 | 389 |
| 389 Err Tokenizer::GetErrorForInvalidToken(const Location& location) const { | 390 Err Tokenizer::GetErrorForInvalidToken(const Location& location) const { |
| 390 std::string help; | 391 std::string help; |
| 391 if (cur_char() == ';') { | 392 if (cur_char() == ';') { |
| 392 // Semicolon. | 393 // Semicolon. |
| 393 help = "Semicolons are not needed, delete this one."; | 394 help = "Semicolons are not needed, delete this one."; |
| 394 } else if (cur_char() == '\t') { | 395 } else if (cur_char() == '\t') { |
| 395 // Tab. | 396 // Tab. |
| 396 help = "You got a tab character in here. Tabs are evil. " | 397 help = "You got a tab character in here. Tabs are evil. " |
| 397 "Convert to spaces."; | 398 "Convert to spaces."; |
| 398 } else if (cur_char() == '/' && cur_ + 1 < input_.size() && | 399 } else if (cur_char() == '/' && cur_ + 1 < input_.size() && |
| 399 (input_[cur_ + 1] == '/' || input_[cur_ + 1] == '*')) { | 400 (input_[cur_ + 1] == '/' || input_[cur_ + 1] == '*')) { |
| 400 // Different types of comments. | 401 // Different types of comments. |
| 401 help = "Comments should start with # instead"; | 402 help = "Comments should start with # instead"; |
| 402 } else if (cur_char() == '\'') { | 403 } else if (cur_char() == '\'') { |
| 403 help = "Strings are delimited by \" characters, not apostrophes."; | 404 help = "Strings are delimited by \" characters, not apostrophes."; |
| 404 } else { | 405 } else { |
| 405 help = "I have no idea what this is."; | 406 help = "I have no idea what this is."; |
| 406 } | 407 } |
| 407 | 408 |
| 408 return Err(location, "Invalid token.", help); | 409 return Err(location, "Invalid token.", help); |
| 409 } | 410 } |
| OLD | NEW |