Experimental scanner fix: recognize one byte strings inside utf16 files.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 178 matching lines @@
     return !(*this == other);
   }
   void Print(const char* prefix) const {
     printf("%s %11s at (%d, %d)",
            prefix, Token::Name(value),
            static_cast<int>(beg), static_cast<int>(end));
     if (literal.size() > 0) {
       for (size_t i = 0; i < literal.size(); i++) {
         printf(is_ascii ? " %02x" : " %04x", literal[i]);
       }
+      printf(" (is ascii: %d)", is_ascii);
     }
     printf(" (last octal start: %d)\n", octal_beg);
   }
 };
 
 
 bool HasLiteral(Token::Value token) {
   return token == Token::IDENTIFIER ||
          token == Token::STRING ||
          token == Token::NUMBER;
@@ skipping 265 matching lines @@
       if (run_experimental) {
         if (benchmark.empty()) benchmark = "Experimental";
         printf("%s(RunTime): %.f ms\n", benchmark.c_str(),
                experimental_total);
       }
     }
   }
   v8::V8::Dispose();
   return 0;
 }