gn: attach comments to parse tree

tools/gn/tokenizer_unittest.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "testing/gtest/include/gtest/gtest.h"
 #include "tools/gn/input_file.h"
 #include "tools/gn/token.h"
 #include "tools/gn/tokenizer.h"
 
 namespace {
@@ skipping 122 matching lines @@
   EXPECT_TRUE(CheckTokenizer("fun(\"foo\") {\nfoo = 12}", fn));
 }
 
 TEST(Tokenizer, Locations) {
   InputFile input(SourceFile("/test"));
   input.SetContents("1 2 \"three\"\n  4");
   Err err;
   std::vector<Token> results = Tokenizer::Tokenize(&input, &err);
 
   ASSERT_EQ(4u, results.size());
-  ASSERT_TRUE(results[0].location() == Location(&input, 1, 1));
-  ASSERT_TRUE(results[1].location() == Location(&input, 1, 3));
-  ASSERT_TRUE(results[2].location() == Location(&input, 1, 5));
-  ASSERT_TRUE(results[3].location() == Location(&input, 2, 3));
+  ASSERT_TRUE(results[0].location() == Location(&input, 1, 1, 1));
+  ASSERT_TRUE(results[1].location() == Location(&input, 1, 3, 3));
+  ASSERT_TRUE(results[2].location() == Location(&input, 1, 5, 5));
+  ASSERT_TRUE(results[3].location() == Location(&input, 2, 3, 8));
 }
 
 TEST(Tokenizer, ByteOffsetOfNthLine) {
   EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1));
 
   // Windows and Posix have different line endings, so check the byte at the
   // location rather than the offset.
   char input1[] = "aaa\nxaa\n\nya";
   EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]);
   EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]);
 
   char input2[3];
   input2[0] = 'a';
   input2[1] = '\n';  // Manually set to avoid Windows double-byte endings.
   input2[2] = 0;
   EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1));
   EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2));
 }
+
+TEST(Tokenizer, Comments) {
+  TokenExpectation fn[] = {
+    { Token::LINE_COMMENT, "# Stuff" },
+    { Token::IDENTIFIER, "fun" },
+    { Token::LEFT_PAREN, "(" },
+    { Token::STRING, "\"foo\"" },
+    { Token::RIGHT_PAREN, ")" },
+    { Token::LEFT_BRACE, "{" },
+    { Token::SUFFIX_COMMENT, "# Things" },
+    { Token::LINE_COMMENT, "#Wee" },
+    { Token::IDENTIFIER, "foo" },
+    { Token::EQUAL, "=" },
+    { Token::INTEGER, "12" },
+    { Token::SUFFIX_COMMENT, "#Zip" },
+    { Token::RIGHT_BRACE, "}" },
+  };
+  EXPECT_TRUE(CheckTokenizer(
+      "# Stuff\nfun(\"foo\") {  # Things\n#Wee\nfoo = 12 #Zip\n}", fn));
+}