gn: attach comments to parse tree

tools/gn/tokenizer.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "tools/gn/tokenizer.h"
 
 #include "base/logging.h"
+#include "base/strings/string_util.h"
 #include "tools/gn/input_file.h"
 
 namespace {
 
 bool CouldBeTwoCharOperatorBegin(char c) {
   return c == '<' || c == '>' || c == '!' || c == '=' || c == '-' ||
          c == '+' || c == '|' || c == '&';
 }
 
 bool CouldBeTwoCharOperatorEnd(char c) {
@@ skipping 82 matching lines @@
     }
     size_t token_begin = cur_;
     AdvanceToEndOfToken(location, type);
     if (has_error())
       break;
     size_t token_end = cur_;
 
     base::StringPiece token_value(&input_.data()[token_begin],
                                   token_end - token_begin);
 
-    if (type == Token::UNCLASSIFIED_OPERATOR)
+    if (type == Token::UNCLASSIFIED_OPERATOR) {
       type = GetSpecificOperatorType(token_value);
-    if (type == Token::IDENTIFIER) {
+    } else if (type == Token::IDENTIFIER) {
       if (token_value == "if")
         type = Token::IF;
       else if (token_value == "else")
         type = Token::ELSE;
       else if (token_value == "true")
         type = Token::TRUE_TOKEN;
       else if (token_value == "false")
         type = Token::FALSE_TOKEN;
+    } else if (type == Token::UNCLASSIFIED_COMMENT) {
+      if (AtStartOfLine(token_begin))
+        type = Token::LINE_COMMENT;
+      else
+        type = Token::SUFFIX_COMMENT;
     }
 
-    // TODO(brettw) This just strips comments from the token stream. This
-    // is probably wrong, they should be removed at a later stage so we can
-    // do things like rewrite the file. But this makes the parser simpler and
-    // is OK for now.
-    if (type != Token::COMMENT)
-      tokens_.push_back(Token(location, type, token_value));
+    tokens_.push_back(Token(location, type, token_value));
   }
   if (err_->has_error())
     tokens_.clear();
   return tokens_;
 }
 
 // static
 size_t Tokenizer::ByteOffsetOfNthLine(const base::StringPiece& buf, int n) {
   DCHECK_GT(n, 0);
 
@@ skipping 53 matching lines @@
     return Token::LEFT_BRACE;
   if (next_char == '}')
     return Token::RIGHT_BRACE;
 
   if (next_char == '.')
     return Token::DOT;
   if (next_char == ',')
     return Token::COMMA;
 
   if (next_char == '#')
-    return Token::COMMENT;
+    return Token::UNCLASSIFIED_COMMENT;
 
   // For the case of '-' differentiate between a negative number and anything
   // else.
   if (next_char == '-') {
     if (!CanIncrement())
       return Token::UNCLASSIFIED_OPERATOR;  // Just the minus before end of
                                             // file.
     char following_char = input_[cur_ + 1];
     if (IsAsciiDigit(following_char))
       return Token::INTEGER;
@@ skipping 65 matching lines @@
     case Token::RIGHT_BRACKET:
     case Token::LEFT_BRACE:
     case Token::RIGHT_BRACE:
     case Token::LEFT_PAREN:
     case Token::RIGHT_PAREN:
     case Token::DOT:
     case Token::COMMA:
       Advance();  // All are one char.
       break;
 
-    case Token::COMMENT:
+    case Token::UNCLASSIFIED_COMMENT:
       // Eat to EOL.
       while (!at_end() && !IsCurrentNewline())
         Advance();
       break;
 
     case Token::INVALID:
     default:
       *err_ = Err(location, "Everything is all messed up",
                   "Please insert system disk in drive A: and press any key.");
       NOTREACHED();
       return;
   }
 }
 
+bool Tokenizer::AtStartOfLine(size_t location) const {
+  while (location > 0) {
+    --location;
+    char c = input_[location];
+    if (c == '\n')
+      return true;
+    if (c != ' ')
+      return false;
+  }
+  return true;
+}
+
 bool Tokenizer::IsCurrentWhitespace() const {
   DCHECK(!at_end());
   char c = input_[cur_];
-  // Note that tab (0x09) is illegal.
-  return c == 0x0A || c == 0x0B || c == 0x0C || c == 0x0D || c == 0x20;
+  // Note that tab (0x09), vertical tab (0x0B), and formfeed (0x0C) are illegal.
+  return c == 0x0A || c == 0x0D || c == 0x20;
 }
 
 bool Tokenizer::IsCurrentStringTerminator(char quote_char) const {
   DCHECK(!at_end());
   if (cur_char() != quote_char)
     return false;
 
   // Check for escaping. \" is not a string terminator, but \\" is. Count
   // the number of preceeding backslashes.
   int num_backslashes = 0;
@@ skipping 14 matching lines @@
   if (IsCurrentNewline()) {
     line_number_++;
     char_in_line_ = 1;
   } else {
     char_in_line_++;
   }
   cur_++;
 }
 
 Location Tokenizer::GetCurrentLocation() const {
-  return Location(input_file_, line_number_, char_in_line_);
+  return Location(
+      input_file_, line_number_, char_in_line_, static_cast<int>(cur_));
 }
 
 Err Tokenizer::GetErrorForInvalidToken(const Location& location) const {
   std::string help;
   if (cur_char() == ';') {
     // Semicolon.
     help = "Semicolons are not needed, delete this one.";
   } else if (cur_char() == '\t') {
     // Tab.
     help = "You got a tab character in here. Tabs are evil. "
            "Convert to spaces.";
   } else if (cur_char() == '/' && cur_ + 1 < input_.size() &&
       (input_[cur_ + 1] == '/' || input_[cur_ + 1] == '*')) {
     // Different types of comments.
     help = "Comments should start with # instead";
   } else {
     help = "I have no idea what this is.";
   }
 
   return Err(location, "Invalid token.", help);
 }