Update the mojom lexer to emit comment tokens.

mojom/mojom_parser/lexer/token_stream.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // TokenStream is the interface between the lexer and the parser. The lexer
 // creates a TokenStream which the parser consumes.
 
 package lexer
 
 type TokenStream interface {
@@ skipping 31 matching lines @@
 }
 
 // See TokenStream.
 func (s *TokenChan) ConsumeNext() {
         if t, open := <-s.tokenChan; open {
                 s.nextToken = t
         } else {
                 s.nextToken = eofToken
         }
 }
+
+// *FilteredTokenStream implements TokenStream
+// This implementation uses an underlying implementation of TokenStream, but
+// it implements a TokenKind blacklist. All tokens of the blacklisted TokenKinds
+// will be silently dropped.
+type FilteredTokenStream struct {
+        // tokenStream is the underlying TokenStream that is the data source.
+        tokenStream TokenStream
+        // blacklist contains the TokenKinds that should be skipped in the source
+        // TokenStream.
+        blacklist map[TokenKind]bool
+}
+
+// See TokenStream.
+func (s *FilteredTokenStream) PeekNext() (token Token) {
+        t := s.tokenStream.PeekNext()
+        for s.isBlacklisted(t) {
+                s.tokenStream.ConsumeNext()
+                t = s.tokenStream.PeekNext()
+        }
+
+        return t
+}
+
+// See TokenStream.
+func (s *FilteredTokenStream) ConsumeNext() {
+        s.tokenStream.ConsumeNext()

[rudominer, 2016/01/20 00:40:31]

I wonder if there should be a ConsumeNextInternal() that is invoked from
PeekNext() and then ConsumeNext() should be implemented as

{
  s.PeekNext()
  s.ConsumeNextInternal()
}

As it is now, if I invoke ConsumeNext() without invoking PeekNext() then it may
not actually consume a (non-filtered) token.

If you do change it as I suggest, then please add an additional test that would
have failed with the current version.

[azani, 2016/01/25 20:37:06]

Done.

+}
+
+// isBlacklisted checks if a Token is of a blacklisted TokenKind.
+func (s *FilteredTokenStream) isBlacklisted(token Token) bool {
+        // We don't allow blacklisting EOF since that would cause an infinite loop.
+        if token.Kind == EOF {

[rudominer, 2016/01/20 00:40:31]

Instead of this, make a constructor for FilteredTokenStream and check there that
EOF is not blacklisted.

[azani, 2016/01/25 20:37:06]

Done.

+                return false
+        }
+        _, ok := s.blacklist[token.Kind]

[rudominer, 2016/01/20 00:40:31]

After the above change you can make this function one line.

[azani, 2016/01/25 20:37:06]

Done.

+        return ok
+}