Update the mojom lexer to emit comment tokens.

mojom/mojom_parser/lexer/token_stream.go

Index: mojom/mojom_parser/lexer/token_stream.go
diff --git a/mojom/mojom_parser/lexer/token_stream.go b/mojom/mojom_parser/lexer/token_stream.go
index beb399a164ca443a0f9293736706e7b1e124be01..9b2ec422afb1eec4f92c8ffde76516fb04eddb35 100644
--- a/mojom/mojom_parser/lexer/token_stream.go
+++ b/mojom/mojom_parser/lexer/token_stream.go
@@ -49,3 +49,38 @@ func (s *TokenChan) ConsumeNext() {
 		s.nextToken = eofToken
 	}
 }
+
+// *FilteredTokenStream implements TokenStream
+// This implementation uses an underlying implementation of TokenStream, but
+// it implements a TokenKind blacklist. All tokens of the blacklisted TokenKinds
+// will be silently dropped.
+type FilteredTokenStream struct {
+	// tokenStream is the underlying TokenStream that is the data source.
+	tokenStream TokenStream
+	// blacklist contains the TokenKinds that should be skipped in the source
+	// TokenStream.
+	blacklist map[TokenKind]bool
+}
+
+// See TokenStream.
+func (s *FilteredTokenStream) PeekNext() (token Token) {
+	for t := s.tokenStream.PeekNext(); s.isBlacklisted(t); s.tokenStream.ConsumeNext() {

[rudominer, 2016/01/16 01:08:58]

Shouldn't there be some condition here for end-of-stream? Can you add a test
with a stream that contains nothing but blacklisted tokens.

[azani, 2016/01/20 00:00:11]

No. isBlacklisted always returns false if the underlying stream returns EOF. But
I was using the for loop wrong, so I've restructured it to make it clearer.

+	}
+
+	return s.tokenStream.PeekNext()
+}
+
+// See TokenStream.
+func (s *FilteredTokenStream) ConsumeNext() {
+	s.tokenStream.ConsumeNext()
+}
+
+// isBlacklisted checks if a Token is of a blacklisted TokenKind.
+func (s *FilteredTokenStream) isBlacklisted(token Token) bool {
+	// We don't allow blacklisting EOF since that would cause an infinite loop.
+	if token.Kind == EOF {
+		return false
+	}
+	_, ok := s.blacklist[token.Kind]
+	return ok
+}