Update the mojom lexer to emit comment tokens.

mojom/mojom_parser/lexer/lexer.go

Index: mojom/mojom_parser/lexer/lexer.go
diff --git a/mojom/mojom_parser/lexer/lexer.go b/mojom/mojom_parser/lexer/lexer.go
index aba64679a49191db49fc9012ac2453cc6606a6c2..2f9d85e6fce4abd02ba4a91234676ecd4a56e0b7 100644
--- a/mojom/mojom_parser/lexer/lexer.go
+++ b/mojom/mojom_parser/lexer/lexer.go
@@ -28,6 +28,15 @@ import (
 // Tokenize accepts a source string and parses it into a stream of tokens which
 // can be read from the returned TokenStream.

[rudominer, 2016/01/16 01:08:58]

You should comment that this function does not return comments.

[azani, 2016/01/20 00:00:11]

Done.

 func Tokenize(source string) TokenStream {
+	return &FilteredTokenStream{
+		tokenizeUnfiltered(source),
+		map[TokenKind]bool{
+			SingleLineComment: true,
+			MultiLineComment:  true}}
+}
+
+// tokenizeUnfiltered does not filter any token.

[rudominer, 2016/01/16 01:08:58]

I would add a slightly longer comment that explains the role this function has
relative to the function Tokenize.

[azani, 2016/01/20 00:00:11]

Done.

+func tokenizeUnfiltered(source string) TokenStream {
 	tokens := make(chan Token)
 	l := lexer{source: source, tokens: tokens}
 	go l.run()
@@ -509,7 +518,7 @@ func lexSingleLineComment(l *lexer) stateFn {
 		l.Consume()
 	}
 
-	l.beginToken()
+	l.emitToken(SingleLineComment)
 	return lexRoot
 }
 
@@ -542,7 +551,7 @@ func lexPossibleEndOfComment(l *lexer) stateFn {
 
 	if l.Peek() == '/' {
 		l.Consume()
-		l.beginToken()
+		l.emitToken(MultiLineComment)
 		return lexRoot
 	}