Update the mojom lexer to emit comment tokens.

mojom/mojom_parser/lexer/lexer_test.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package lexer
 
 import (
         "strings"
         "testing"
 )
@@ skipping 62 matching lines @@
                 {"10e+5", FloatConst},
                 {"10e-5", FloatConst},
                 {"3.14E5", FloatConst},
                 {"3.14e5", FloatConst},
                 {"3.14E+55", FloatConst},
                 {"3.14e+55", FloatConst},
                 {"3.14E-55", FloatConst},
                 {"3.14e-55", FloatConst},
                 {"\"hello world\"", StringLiteral},
                 {"\"hello \\\"real\\\" world\"", StringLiteral},
+                {"// hello comment world", SingleLineComment},
+                {"/* hello \n comment */", MultiLineComment},
         }
 
         for i := range testData {
                 l := lexer{source: testData[i].source, tokens: make(chan Token)}
                 go l.run()
                 tokens := pumpTokens(l.tokens)
 
                 if len(tokens) != 1 {
                         t.Fatalf("Source('%v'): Expected 1 token but got %v instead: %v",
                                 testData[i].source, len(tokens), tokens)
@@ skipping 86 matching lines @@
         checkEq(t, ErrorUnterminatedStringLiteral, ts.PeekNext().Kind)
 }
 
 // TestUnterminatedStringLiteralEol tests that the correct error is emitted if
 // a quoted string is closed on a subsequent line.
 func TestUnterminatedStringLiteralEol(t *testing.T) {
         ts := Tokenize("\"hello\n world\"")
         checkEq(t, ErrorUnterminatedStringLiteral, ts.PeekNext().Kind)
 }
 
-// TestSingleLineComment tests that single line comments are correctly skipped.
+// TestSingleLineComment tests that single line comments are correctly lexed.

[rudominer, 2016/01/16 01:08:58]

Don't you want to test both that they are skipped when they should be and not
skipped otherwise?

[azani, 2016/01/20 00:00:11]

Done.

 func TestSingleLineComment(t *testing.T) {
-        ts := Tokenize("( // some stuff\n)")
+        ts := tokenizeUnfiltered("( // some stuff\n)")
         checkEq(t, LParen, ts.PeekNext().Kind)
         ts.ConsumeNext()
+        checkEq(t, SingleLineComment, ts.PeekNext().Kind)
+        checkEq(t, "// some stuff", ts.PeekNext().Text)
+        ts.ConsumeNext()
         checkEq(t, RParen, ts.PeekNext().Kind)
 }
 
 // TestMultiLineComment tests that multi line comments are correctly skipped.

[rudominer, 2016/01/16 01:08:58]

skipped?

[azani, 2016/01/20 00:00:11]

Done.

 func TestMultiLineComment(t *testing.T) {
-        ts := Tokenize("( /* hello world/  * *\n */)")
+        ts := tokenizeUnfiltered("( /* hello world/  * *\n */)")
         checkEq(t, LParen, ts.PeekNext().Kind)
         ts.ConsumeNext()
+        checkEq(t, MultiLineComment, ts.PeekNext().Kind)
+        checkEq(t, "/* hello world/  * *\n */", ts.PeekNext().Text)
+        ts.ConsumeNext()
         checkEq(t, RParen, ts.PeekNext().Kind)
 }
 
 // TestUnterminatedMultiLineComment tests that unterminated multiline comments
 // emit the correct error.
 func TestUnterminatedMultiLineComment(t *testing.T) {
         ts := Tokenize("( /* hello world/  * *\n )")
         checkEq(t, LParen, ts.PeekNext().Kind)
         ts.ConsumeNext()
         checkEq(t, ErrorUnterminatedComment, ts.PeekNext().Kind)
@@ skipping 54 matching lines @@
 
 // TestTokenSnippetLongPrelude tests snippet generation in the presence of a
 // long line with many characters preceeding the token.
 func TestTokenSnippetLongPrelude(t *testing.T) {
         source := strings.Repeat(" ", 70) + "hello world"
         ts := Tokenize(source)
         expected := strings.Repeat(" ", 58) + "hello world\n"
         expected += strings.Repeat(" ", 58) + "^^^^^"
         checkEq(t, expected, ts.PeekNext().Snippet(source, false))
 }