Script streaming: UTF-8 handling fix.

src/scanner-character-streams.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/scanner-character-streams.h"
 
 #include "include/v8.h"
 #include "src/handles.h"
@@ skipping 402 matching lines @@
   // current chunk to utf8_split_char_buffer_. They will be converted when the
   // next data chunk arrives. Note that all valid UTF-8 characters are at most 4
   // bytes long, but if the data is invalid, we can have character values bigger
   // than unibrow::Utf8::kMaxOneByteChar for more than 4 consecutive bytes.
   while (current_data_length_ > current_data_offset_ &&
          (c = current_data_[current_data_length_ - 1]) >
              unibrow::Utf8::kMaxOneByteChar &&
          utf8_split_char_buffer_length_ < 4) {
     --current_data_length_;
     ++utf8_split_char_buffer_length_;
+    if (c >= (3 << 6)) {
+      // 3 << 6 = 0b11000000; this is the first byte of the multi-byte
+      // character. No need to copy the previous characters into the conversion
+      // buffer (even if they're multi-byte).
+      break;
+    }
   }
   CHECK(utf8_split_char_buffer_length_ <= 4);
   for (unsigned i = 0; i < utf8_split_char_buffer_length_; ++i) {
     utf8_split_char_buffer_[i] = current_data_[current_data_length_ + i];
   }
 }
 
 
 // ----------------------------------------------------------------------------
 // ExternalTwoByteStringUtf16CharacterStream
 
 ExternalTwoByteStringUtf16CharacterStream::
     ~ExternalTwoByteStringUtf16CharacterStream() { }
 
 
 ExternalTwoByteStringUtf16CharacterStream
     ::ExternalTwoByteStringUtf16CharacterStream(
         Handle<ExternalTwoByteString> data,
         int start_position,
         int end_position)
     : Utf16CharacterStream(),
       source_(data),
       raw_data_(data->GetTwoByteData(start_position)) {
   buffer_cursor_ = raw_data_,
   buffer_end_ = raw_data_ + (end_position - start_position);
   pos_ = start_position;
 }
 
 } }  // namespace v8::internal