gn: attach comments to parse tree

tools/gn/token.h

Index: tools/gn/token.h
diff --git a/tools/gn/token.h b/tools/gn/token.h
index 8c7cd58f1e54d52cd579401e010b6d0a7c6a4bc3..9b8dd5d2e9869995f981fd82b798d78195eddee6 100644
--- a/tools/gn/token.h
+++ b/tools/gn/token.h
@@ -45,9 +45,12 @@ class Token {
     ELSE,
     IDENTIFIER, // foo
     COMMA,  // ,
-    COMMENT,    // #...\n
+    UNCLASSIFIED_COMMENT,  // #...\n, of unknown style (will be converted
+                           // to one below)
+    LINE_COMMENT,      // #...\n on a line alone.
+    SUFFIX_COMMENT,    // #...\n on a line following other code.
 
-    UNCLASSIFIED_OPERATOR,  // TODO(scottmg): This shouldn't be necessary.
+    UNCLASSIFIED_OPERATOR,
 
     NUM_TYPES
   };
@@ -59,10 +62,12 @@ class Token {
   const base::StringPiece& value() const { return value_; }
   const Location& location() const { return location_; }
   LocationRange range() const {
-    return LocationRange(location_,
-                         Location(location_.file(), location_.line_number(),
-                                  location_.char_offset() +
-                                      static_cast<int>(value_.size())));
+    return LocationRange(
+        location_,
+        Location(location_.file(),
+                 location_.line_number(),
+                 location_.char_offset() + static_cast<int>(value_.size()),
+                 location_.byte() + static_cast<int>(value_.size())));
   }
 
   // Helper functions for comparing this token to something.