Experimental scanner fix: recognize one byte strings inside utf16 files.

src/lexer/experimental-scanner.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 100 matching lines @@
 bool ExperimentalScanner<uint16_t>::FillLiteral(
     const TokenDesc& token, LiteralDesc* literal) {
   literal->beg_pos = token.beg_pos;
   const uint16_t* start = buffer_ + token.beg_pos;
   const uint16_t* end = buffer_ + token.end_pos;
   if (token.token == Token::STRING) {
     ++start;
     --end;
   }
   if (!token.has_escapes) {
-    literal->is_ascii = false;  // FIXME: utf16 can contain only ascii chars.
+    // UTF-16 can also contain only one byte chars. Note that is_ascii here
+    // means is_onebyte.
+    literal->is_ascii = true;
+    literal->buffer.Reset();
+    for (const uint16_t* cursor = start; cursor != end; ++cursor) {
+      if (*cursor >= unibrow::Latin1::kMaxChar) {
+        literal->is_ascii = false;
+        break;
+      }
+      literal->buffer.AddChar(*cursor);
+    }
     literal->length = end - start;
-    literal->utf16_string = Vector<const uint16_t>(start, literal->length);
+    if (literal->is_ascii) {
+      literal->ascii_string = literal->buffer.ascii_literal();
+    } else {
+      literal->buffer.Reset();
+      literal->utf16_string = Vector<const uint16_t>(start, literal->length);
+    }
     return true;
   }
   literal->buffer.Reset();
   for (const uint16_t* cursor = start; cursor != end;) {
     if (*cursor != '\\') {
       literal->buffer.AddChar(*cursor++);
     } else if (token.token == Token::IDENTIFIER) {
       uc32 c;
       cursor = ScanIdentifierUnicodeEscape(cursor, end, &c);
       ASSERT(cursor != NULL);
@@ skipping 18 matching lines @@
 template<>
 bool ExperimentalScanner<int8_t>::FillLiteral(
     const TokenDesc& token, LiteralDesc* literal) {
   // FIXME: implement.
   return false;
 }
 
 
 }
 }