Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(2)

Side by Side Diff: tools/gn/tokenizer.cc

Issue 1525183002: tools/gn: rename char_offset to column_number (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: fix indentation Created 5 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/gn/tokenizer.h ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "tools/gn/tokenizer.h" 5 #include "tools/gn/tokenizer.h"
6 6
7 #include "base/logging.h" 7 #include "base/logging.h"
8 #include "base/strings/string_util.h" 8 #include "base/strings/string_util.h"
9 #include "tools/gn/input_file.h" 9 #include "tools/gn/input_file.h"
10 10
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after
67 } 67 }
68 68
69 } // namespace 69 } // namespace
70 70
71 Tokenizer::Tokenizer(const InputFile* input_file, Err* err) 71 Tokenizer::Tokenizer(const InputFile* input_file, Err* err)
72 : input_file_(input_file), 72 : input_file_(input_file),
73 input_(input_file->contents()), 73 input_(input_file->contents()),
74 err_(err), 74 err_(err),
75 cur_(0), 75 cur_(0),
76 line_number_(1), 76 line_number_(1),
77 char_in_line_(1) { 77 column_number_(1) {
78 } 78 }
79 79
80 Tokenizer::~Tokenizer() { 80 Tokenizer::~Tokenizer() {
81 } 81 }
82 82
83 // static 83 // static
84 std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) { 84 std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) {
85 Tokenizer t(input_file, err); 85 Tokenizer t(input_file, err);
86 return t.Run(); 86 return t.Run();
87 } 87 }
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
119 type = Token::TRUE_TOKEN; 119 type = Token::TRUE_TOKEN;
120 else if (token_value == "false") 120 else if (token_value == "false")
121 type = Token::FALSE_TOKEN; 121 type = Token::FALSE_TOKEN;
122 } else if (type == Token::UNCLASSIFIED_COMMENT) { 122 } else if (type == Token::UNCLASSIFIED_COMMENT) {
123 if (AtStartOfLine(token_begin) && 123 if (AtStartOfLine(token_begin) &&
124 // If it's a standalone comment, but is a continuation of a comment on 124 // If it's a standalone comment, but is a continuation of a comment on
125 // a previous line, then instead make it a continued suffix comment. 125 // a previous line, then instead make it a continued suffix comment.
126 (tokens_.empty() || tokens_.back().type() != Token::SUFFIX_COMMENT || 126 (tokens_.empty() || tokens_.back().type() != Token::SUFFIX_COMMENT ||
127 tokens_.back().location().line_number() + 1 != 127 tokens_.back().location().line_number() + 1 !=
128 location.line_number() || 128 location.line_number() ||
129 tokens_.back().location().char_offset() != location.char_offset())) { 129 tokens_.back().location().column_number() !=
130 location.column_number())) {
130 type = Token::LINE_COMMENT; 131 type = Token::LINE_COMMENT;
131 if (!at_end()) // Could be EOF. 132 if (!at_end()) // Could be EOF.
132 Advance(); // The current \n. 133 Advance(); // The current \n.
133 // If this comment is separated from the next syntax element, then we 134 // If this comment is separated from the next syntax element, then we
134 // want to tag it as a block comment. This will become a standalone 135 // want to tag it as a block comment. This will become a standalone
135 // statement at the parser level to keep this comment separate, rather 136 // statement at the parser level to keep this comment separate, rather
136 // than attached to the subsequent statement. 137 // than attached to the subsequent statement.
137 while (!at_end() && IsCurrentWhitespace()) { 138 while (!at_end() && IsCurrentWhitespace()) {
138 if (IsCurrentNewline()) { 139 if (IsCurrentNewline()) {
139 type = Token::BLOCK_COMMENT; 140 type = Token::BLOCK_COMMENT;
(...skipping 227 matching lines...) Expand 10 before | Expand all | Expand 10 after
367 } 368 }
368 369
369 bool Tokenizer::IsCurrentNewline() const { 370 bool Tokenizer::IsCurrentNewline() const {
370 return IsNewline(input_, cur_); 371 return IsNewline(input_, cur_);
371 } 372 }
372 373
373 void Tokenizer::Advance() { 374 void Tokenizer::Advance() {
374 DCHECK(cur_ < input_.size()); 375 DCHECK(cur_ < input_.size());
375 if (IsCurrentNewline()) { 376 if (IsCurrentNewline()) {
376 line_number_++; 377 line_number_++;
377 char_in_line_ = 1; 378 column_number_ = 1;
378 } else { 379 } else {
379 char_in_line_++; 380 column_number_++;
380 } 381 }
381 cur_++; 382 cur_++;
382 } 383 }
383 384
384 Location Tokenizer::GetCurrentLocation() const { 385 Location Tokenizer::GetCurrentLocation() const {
385 return Location( 386 return Location(
386 input_file_, line_number_, char_in_line_, static_cast<int>(cur_)); 387 input_file_, line_number_, column_number_, static_cast<int>(cur_));
387 } 388 }
388 389
389 Err Tokenizer::GetErrorForInvalidToken(const Location& location) const { 390 Err Tokenizer::GetErrorForInvalidToken(const Location& location) const {
390 std::string help; 391 std::string help;
391 if (cur_char() == ';') { 392 if (cur_char() == ';') {
392 // Semicolon. 393 // Semicolon.
393 help = "Semicolons are not needed, delete this one."; 394 help = "Semicolons are not needed, delete this one.";
394 } else if (cur_char() == '\t') { 395 } else if (cur_char() == '\t') {
395 // Tab. 396 // Tab.
396 help = "You got a tab character in here. Tabs are evil. " 397 help = "You got a tab character in here. Tabs are evil. "
397 "Convert to spaces."; 398 "Convert to spaces.";
398 } else if (cur_char() == '/' && cur_ + 1 < input_.size() && 399 } else if (cur_char() == '/' && cur_ + 1 < input_.size() &&
399 (input_[cur_ + 1] == '/' || input_[cur_ + 1] == '*')) { 400 (input_[cur_ + 1] == '/' || input_[cur_ + 1] == '*')) {
400 // Different types of comments. 401 // Different types of comments.
401 help = "Comments should start with # instead"; 402 help = "Comments should start with # instead";
402 } else if (cur_char() == '\'') { 403 } else if (cur_char() == '\'') {
403 help = "Strings are delimited by \" characters, not apostrophes."; 404 help = "Strings are delimited by \" characters, not apostrophes.";
404 } else { 405 } else {
405 help = "I have no idea what this is."; 406 help = "I have no idea what this is.";
406 } 407 }
407 408
408 return Err(location, "Invalid token.", help); 409 return Err(location, "Invalid token.", help);
409 } 410 }
OLDNEW
« no previous file with comments | « tools/gn/tokenizer.h ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698