Experimental scanner: fix two-byte handling in lexer-shell.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 277 matching lines @@
             &experimental_tokens, repeat, harmony_settings);
         break;
       case UTF16:
       case UTF8TO16: {
         const uc16* buffer_16 = reinterpret_cast<const uc16*>(buffer);
         const uc16* buffer_end_16 = reinterpret_cast<const uc16*>(buffer_end);
         source = isolate->factory()->NewStringFromTwoByte(
             Vector<const uc16>(buffer_16, buffer_end_16 - buffer_16));
         // If the string was just an expaneded one byte string, V8 detects it
         // and doesn't store it as two byte.
-        CHECK(source->IsTwoByteRepresentation());
-        experimental_time = RunExperimentalScanner<uint16_t>(
-            source, isolate, encoding, print_tokens || check_tokens,
-            &experimental_tokens, repeat, harmony_settings);
+        if (!source->IsTwoByteRepresentation()) {
+          experimental_time = RunExperimentalScanner<uint8_t>(
+              source, isolate, encoding, print_tokens || check_tokens,
+              &experimental_tokens, repeat, harmony_settings);
+        } else {
+          experimental_time = RunExperimentalScanner<uint16_t>(
+              source, isolate, encoding, print_tokens || check_tokens,
+              &experimental_tokens, repeat, harmony_settings);
+        }
         break;
       }
       default:
         printf("Encoding not supported by the experimental scanner\n");
         exit(1);
         break;
     }
   }
   if (print_tokens && !run_experimental) {
     PrintTokens("Baseline", baseline_tokens);
@@ skipping 113 matching lines @@
       if (run_experimental) {
         if (benchmark.empty()) benchmark = "Experimental";
         printf("%s(RunTime): %.f ms\n", benchmark.c_str(),
                experimental_total);
       }
     }
   }
   v8::V8::Dispose();
   return 0;
 }