| Index: src/scanner.cc
|
| diff --git a/src/scanner.cc b/src/scanner.cc
|
| index c6c0a8d6a23448b7be244224ec48bc5b37b16cee..f6c10b44034e02c5ee3a801fa3f9264ea7049937 100644
|
| --- a/src/scanner.cc
|
| +++ b/src/scanner.cc
|
| @@ -238,6 +238,11 @@ Token::Value Scanner::Next() {
|
| next_.location.end_pos = current_.location.end_pos;
|
| }
|
| current_ = next_;
|
| + if (next_next_.token != Token::UNINITIALIZED) {
|
| + next_ = next_next_;
|
| + next_next_.token = Token::UNINITIALIZED;
|
| + return current_.token;
|
| + }
|
| has_line_terminator_before_next_ = false;
|
| has_multiline_comment_before_next_ = false;
|
| if (static_cast<unsigned>(c0_) <= 0x7f) {
|
| @@ -256,6 +261,20 @@ Token::Value Scanner::Next() {
|
| }
|
|
|
|
|
| +Token::Value Scanner::PeekAhead() {
|
| + if (next_next_.token != Token::UNINITIALIZED) {
|
| + return next_next_.token;
|
| + }
|
| + TokenDesc prev = current_;
|
| + Next();
|
| + Token::Value ret = next_.token;
|
| + next_next_ = next_;
|
| + next_ = current_;
|
| + current_ = prev;
|
| + return ret;
|
| +}
|
| +
|
| +
|
| // TODO(yangguo): check whether this is actually necessary.
|
| static inline bool IsLittleEndianByteOrderMark(uc32 c) {
|
| // The Unicode value U+FFFE is guaranteed never to be assigned as a
|
|
|