Update the mojom lexer to emit comment tokens.

mojom/mojom_parser/lexer/lexer_test.go

Index: mojom/mojom_parser/lexer/lexer_test.go
diff --git a/mojom/mojom_parser/lexer/lexer_test.go b/mojom/mojom_parser/lexer/lexer_test.go
index fa7d8b8261450a44df9be7bb77c2848b91b0811e..2778c6861e2d39a62516a4b9af09b199972d50eb 100644
--- a/mojom/mojom_parser/lexer/lexer_test.go
+++ b/mojom/mojom_parser/lexer/lexer_test.go
@@ -80,6 +80,8 @@ func TestAllSingleTokens(t *testing.T) {
 		{"3.14e-55", FloatConst},
 		{"\"hello world\"", StringLiteral},
 		{"\"hello \\\"real\\\" world\"", StringLiteral},
+		{"// hello comment world", SingleLineComment},
+		{"/* hello \n comment */", MultiLineComment},
 	}
 
 	for i := range testData {
@@ -186,19 +188,25 @@ func TestUnterminatedStringLiteralEol(t *testing.T) {
 	checkEq(t, ErrorUnterminatedStringLiteral, ts.PeekNext().Kind)
 }
 
-// TestSingleLineComment tests that single line comments are correctly skipped.
+// TestSingleLineComment tests that single line comments are correctly lexed.

[rudominer, 2016/01/16 01:08:58]

Don't you want to test both that they are skipped when they should be and not
skipped otherwise?

[azani, 2016/01/20 00:00:11]

Done.

 func TestSingleLineComment(t *testing.T) {
-	ts := Tokenize("( // some stuff\n)")
+	ts := tokenizeUnfiltered("( // some stuff\n)")
 	checkEq(t, LParen, ts.PeekNext().Kind)
 	ts.ConsumeNext()
+	checkEq(t, SingleLineComment, ts.PeekNext().Kind)
+	checkEq(t, "// some stuff", ts.PeekNext().Text)
+	ts.ConsumeNext()
 	checkEq(t, RParen, ts.PeekNext().Kind)
 }
 
 // TestMultiLineComment tests that multi line comments are correctly skipped.

[rudominer, 2016/01/16 01:08:58]

skipped?

[azani, 2016/01/20 00:00:11]

Done.

 func TestMultiLineComment(t *testing.T) {
-	ts := Tokenize("( /* hello world/  * *\n */)")
+	ts := tokenizeUnfiltered("( /* hello world/  * *\n */)")
 	checkEq(t, LParen, ts.PeekNext().Kind)
 	ts.ConsumeNext()
+	checkEq(t, MultiLineComment, ts.PeekNext().Kind)
+	checkEq(t, "/* hello world/  * *\n */", ts.PeekNext().Text)
+	ts.ConsumeNext()
 	checkEq(t, RParen, ts.PeekNext().Kind)
 }