New lexer for mojom written in go.

mojom/mojom_parser/lexer/token_stream.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// TokenStream is the interface between the lexer and the parser. The lexer
+// creates a TokenStream which the parser consumes.
+
+package lexer
+
+type TokenStream interface {
+        // Returns the next Token in the stream without advancing the cursor,
+        // or returns the EOF token if the cursor is already past the end.
+        PeekNext() Token
+
+        // Advances the cursor in the stream or does nothing if the cursor is
+        // already past the end of the stream
+        ConsumeNext()
+}
+
+// The EOF token is returned by TokenStream to signal the end of the stream.
+var eofToken = Token{Kind: EOF}
+
+// *TokenChan implements TokenStream.
+// This implementation uses a non-buffered channel to pass the tokens from the
+// lexer to the parser. One end of the channel is held by the lexer and the
+// other is in the TokenChan object that is passed to the parser.
+type TokenChan struct {
+        tokenChan chan Token
+        nextToken Token
+        // read is true if a token has been read out of the channel into nextToken.
+        read bool
+}
+
+// See TokenStream.
+func (s *TokenChan) PeekNext() (token Token) {
+        if !s.read {
+                s.read = true
+                s.ConsumeNext()
+        }
+
+        return s.nextToken
+}
+
+// See TokenStream.
+func (s *TokenChan) ConsumeNext() {
+        if t, open := <-s.tokenChan; open {
+                s.nextToken = t
+        } else {
+                s.nextToken = eofToken
+        }
+}