Update the mojom lexer to emit comment tokens.

mojom/mojom_parser/lexer/lexer.go

Index: mojom/mojom_parser/lexer/lexer.go
diff --git a/mojom/mojom_parser/lexer/lexer.go b/mojom/mojom_parser/lexer/lexer.go
index aba64679a49191db49fc9012ac2453cc6606a6c2..76f4273df01a594e3919837f9be571959d335074 100644
--- a/mojom/mojom_parser/lexer/lexer.go
+++ b/mojom/mojom_parser/lexer/lexer.go
@@ -26,8 +26,18 @@ import (
 )
 
 // Tokenize accepts a source string and parses it into a stream of tokens which
-// can be read from the returned TokenStream.
+// can be read from the returned TokenStream. Comment tokens are ommitted from
+// the returned stream.
 func Tokenize(source string) TokenStream {
+	return NewFilteredTokenStream(
+		tokenizeUnfiltered(source),
+		[]TokenKind{SingleLineComment, MultiLineComment})
+}
+
+// tokenizeUnfiltered returns a TokenStream which does not filter out any of the
+// tokens in the channel. It is used for testing and by Tokenize which adds a
+// filter on top of the stream returned by tokenizeUnfiltered.
+func tokenizeUnfiltered(source string) TokenStream {
 	tokens := make(chan Token)
 	l := lexer{source: source, tokens: tokens}
 	go l.run()
@@ -509,7 +519,7 @@ func lexSingleLineComment(l *lexer) stateFn {
 		l.Consume()
 	}
 
-	l.beginToken()
+	l.emitToken(SingleLineComment)
 	return lexRoot
 }
 
@@ -542,7 +552,7 @@ func lexPossibleEndOfComment(l *lexer) stateFn {
 
 	if l.Peek() == '/' {
 		l.Consume()
-		l.beginToken()
+		l.emitToken(MultiLineComment)
 		return lexRoot
 	}