OLD | NEW |
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "tools/gn/tokenizer.h" | 5 #include "tools/gn/tokenizer.h" |
6 | 6 |
7 #include "base/logging.h" | 7 #include "base/logging.h" |
8 #include "base/strings/string_util.h" | 8 #include "base/strings/string_util.h" |
9 #include "tools/gn/input_file.h" | 9 #include "tools/gn/input_file.h" |
10 | 10 |
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
67 } | 67 } |
68 | 68 |
69 } // namespace | 69 } // namespace |
70 | 70 |
71 Tokenizer::Tokenizer(const InputFile* input_file, Err* err) | 71 Tokenizer::Tokenizer(const InputFile* input_file, Err* err) |
72 : input_file_(input_file), | 72 : input_file_(input_file), |
73 input_(input_file->contents()), | 73 input_(input_file->contents()), |
74 err_(err), | 74 err_(err), |
75 cur_(0), | 75 cur_(0), |
76 line_number_(1), | 76 line_number_(1), |
77 char_in_line_(1) { | 77 column_number_(1) { |
78 } | 78 } |
79 | 79 |
80 Tokenizer::~Tokenizer() { | 80 Tokenizer::~Tokenizer() { |
81 } | 81 } |
82 | 82 |
83 // static | 83 // static |
84 std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) { | 84 std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) { |
85 Tokenizer t(input_file, err); | 85 Tokenizer t(input_file, err); |
86 return t.Run(); | 86 return t.Run(); |
87 } | 87 } |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
119 type = Token::TRUE_TOKEN; | 119 type = Token::TRUE_TOKEN; |
120 else if (token_value == "false") | 120 else if (token_value == "false") |
121 type = Token::FALSE_TOKEN; | 121 type = Token::FALSE_TOKEN; |
122 } else if (type == Token::UNCLASSIFIED_COMMENT) { | 122 } else if (type == Token::UNCLASSIFIED_COMMENT) { |
123 if (AtStartOfLine(token_begin) && | 123 if (AtStartOfLine(token_begin) && |
124 // If it's a standalone comment, but is a continuation of a comment on | 124 // If it's a standalone comment, but is a continuation of a comment on |
125 // a previous line, then instead make it a continued suffix comment. | 125 // a previous line, then instead make it a continued suffix comment. |
126 (tokens_.empty() || tokens_.back().type() != Token::SUFFIX_COMMENT || | 126 (tokens_.empty() || tokens_.back().type() != Token::SUFFIX_COMMENT || |
127 tokens_.back().location().line_number() + 1 != | 127 tokens_.back().location().line_number() + 1 != |
128 location.line_number() || | 128 location.line_number() || |
129 tokens_.back().location().char_offset() != location.char_offset())) { | 129 tokens_.back().location().column_number() != |
| 130 location.column_number())) { |
130 type = Token::LINE_COMMENT; | 131 type = Token::LINE_COMMENT; |
131 if (!at_end()) // Could be EOF. | 132 if (!at_end()) // Could be EOF. |
132 Advance(); // The current \n. | 133 Advance(); // The current \n. |
133 // If this comment is separated from the next syntax element, then we | 134 // If this comment is separated from the next syntax element, then we |
134 // want to tag it as a block comment. This will become a standalone | 135 // want to tag it as a block comment. This will become a standalone |
135 // statement at the parser level to keep this comment separate, rather | 136 // statement at the parser level to keep this comment separate, rather |
136 // than attached to the subsequent statement. | 137 // than attached to the subsequent statement. |
137 while (!at_end() && IsCurrentWhitespace()) { | 138 while (!at_end() && IsCurrentWhitespace()) { |
138 if (IsCurrentNewline()) { | 139 if (IsCurrentNewline()) { |
139 type = Token::BLOCK_COMMENT; | 140 type = Token::BLOCK_COMMENT; |
(...skipping 227 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
367 } | 368 } |
368 | 369 |
369 bool Tokenizer::IsCurrentNewline() const { | 370 bool Tokenizer::IsCurrentNewline() const { |
370 return IsNewline(input_, cur_); | 371 return IsNewline(input_, cur_); |
371 } | 372 } |
372 | 373 |
373 void Tokenizer::Advance() { | 374 void Tokenizer::Advance() { |
374 DCHECK(cur_ < input_.size()); | 375 DCHECK(cur_ < input_.size()); |
375 if (IsCurrentNewline()) { | 376 if (IsCurrentNewline()) { |
376 line_number_++; | 377 line_number_++; |
377 char_in_line_ = 1; | 378 column_number_ = 1; |
378 } else { | 379 } else { |
379 char_in_line_++; | 380 column_number_++; |
380 } | 381 } |
381 cur_++; | 382 cur_++; |
382 } | 383 } |
383 | 384 |
384 Location Tokenizer::GetCurrentLocation() const { | 385 Location Tokenizer::GetCurrentLocation() const { |
385 return Location( | 386 return Location( |
386 input_file_, line_number_, char_in_line_, static_cast<int>(cur_)); | 387 input_file_, line_number_, column_number_, static_cast<int>(cur_)); |
387 } | 388 } |
388 | 389 |
389 Err Tokenizer::GetErrorForInvalidToken(const Location& location) const { | 390 Err Tokenizer::GetErrorForInvalidToken(const Location& location) const { |
390 std::string help; | 391 std::string help; |
391 if (cur_char() == ';') { | 392 if (cur_char() == ';') { |
392 // Semicolon. | 393 // Semicolon. |
393 help = "Semicolons are not needed, delete this one."; | 394 help = "Semicolons are not needed, delete this one."; |
394 } else if (cur_char() == '\t') { | 395 } else if (cur_char() == '\t') { |
395 // Tab. | 396 // Tab. |
396 help = "You got a tab character in here. Tabs are evil. " | 397 help = "You got a tab character in here. Tabs are evil. " |
397 "Convert to spaces."; | 398 "Convert to spaces."; |
398 } else if (cur_char() == '/' && cur_ + 1 < input_.size() && | 399 } else if (cur_char() == '/' && cur_ + 1 < input_.size() && |
399 (input_[cur_ + 1] == '/' || input_[cur_ + 1] == '*')) { | 400 (input_[cur_ + 1] == '/' || input_[cur_ + 1] == '*')) { |
400 // Different types of comments. | 401 // Different types of comments. |
401 help = "Comments should start with # instead"; | 402 help = "Comments should start with # instead"; |
402 } else if (cur_char() == '\'') { | 403 } else if (cur_char() == '\'') { |
403 help = "Strings are delimited by \" characters, not apostrophes."; | 404 help = "Strings are delimited by \" characters, not apostrophes."; |
404 } else { | 405 } else { |
405 help = "I have no idea what this is."; | 406 help = "I have no idea what this is."; |
406 } | 407 } |
407 | 408 |
408 return Err(location, "Invalid token.", help); | 409 return Err(location, "Invalid token.", help); |
409 } | 410 } |
OLD | NEW |