Update the mojom lexer to emit comment tokens.

mojom/mojom_parser/lexer/lexer.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // To use the lexer, call Tokenize with the source string to obtain
 // a TokenStream. The lexer is run concurrently so you should be able to
 // use the TokenStream before the lexer is done with the source.
 //
 // The lexer is implemented as a state machine. The states are represented
 // by functions (the stateFn type) which accept a lexer and return the
 // new state.
 //
 // Most states also have an isFooStart function which helps determine if
 // a transition to Foo is appropriate. Those functions accept a single
 // rune as a parameter and return true if the state machine should
 // transition to state Foo. Some states do not have such functions on
 // account of the transition condition being trivial.
 //
 // The lexer implementation was inspired by
 // http://cuddle.googlecode.com/hg/talk/lex.html
 
 package lexer
 
 import (
         "unicode/utf8"
 )
 
 // Tokenize accepts a source string and parses it into a stream of tokens which
-// can be read from the returned TokenStream.
+// can be read from the returned TokenStream. Comment tokens are ommitted from
+// the returned stream.
 func Tokenize(source string) TokenStream {
+        return NewFilteredTokenStream(
+                tokenizeUnfiltered(source),
+                []TokenKind{SingleLineComment, MultiLineComment})
+}
+
+// tokenizeUnfiltered returns a TokenStream which does not filter out any of the
+// tokens in the channel. It is used for testing and by Tokenize which adds a
+// filter on top of the stream returned by tokenizeUnfiltered.
+func tokenizeUnfiltered(source string) TokenStream {
         tokens := make(chan Token)
         l := lexer{source: source, tokens: tokens}
         go l.run()
         return &TokenChan{tokenChan: tokens}
 }
 
 type lexer struct {
         // source is the source code to be lexed.
         source string
 
@@ skipping 461 matching lines @@
 
 // lexSingleLineComment consumes a single line comment.
 func lexSingleLineComment(l *lexer) stateFn {
         // Consume the '/'
         l.Consume()
 
         for !l.IsEos() && l.Peek() != '\n' {
                 l.Consume()
         }
 
-        l.beginToken()
+        l.emitToken(SingleLineComment)
         return lexRoot
 }
 
 // lexMultiLineComment consumes a multi-line comment.
 func lexMultiLineComment(l *lexer) stateFn {
         // Consume the '*'.
         l.Consume()
 
         for !l.IsEos() {
                 if l.Peek() == '*' {
@@ skipping 12 matching lines @@
         // Consume the '*'
         l.Consume()
 
         if l.IsEos() {
                 l.emitToken(ErrorUnterminatedComment)
                 return nil
         }
 
         if l.Peek() == '/' {
                 l.Consume()
-                l.beginToken()
+                l.emitToken(MultiLineComment)
                 return lexRoot
         }
 
         return lexMultiLineComment
 }