Experimental lexer: track one-byteness on the fly.

src/lexer/experimental-scanner.h

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 220 matching lines @@
     return literal.length() == keyword.length() &&
         (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
 
  protected:
   struct TokenDesc {
     Token::Value token;
     int beg_pos;
     int end_pos;
     bool has_escapes;
+    bool is_onebyte;
   };
 
   struct LiteralDesc {
     int beg_pos;
     bool is_ascii;
     int length;
     Vector<const char> ascii_string;
     Vector<const uc16> utf16_string;
     LiteralBuffer buffer;
     LiteralDesc() : beg_pos(-1), is_ascii(false), length(0) { }
@@ skipping 355 matching lines @@
   // character.
   const Char* temp_cursor = last_octal_end_ - 1;
   while (temp_cursor >= buffer_ && *temp_cursor >= '0' && *temp_cursor <= '7')
     --temp_cursor;
   return Location(temp_cursor - buffer_ + 1, last_octal_end_ - buffer_);
 }
 
 } }
 
 #endif  // V8_LEXER_EXPERIMENTAL_SCANNER_H