Script streaming: more UTF-8 handing fixes (again).

src/scanner-character-streams.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/scanner-character-streams.h"
 
 #include "include/v8.h"
 #include "src/handles.h"
 #include "src/unicode-inl.h"
 
 namespace v8 {
 namespace internal {
 
 namespace {
 
 unsigned CopyCharsHelper(uint16_t* dest, unsigned length, const uint8_t* src,
                          unsigned* src_pos, unsigned src_length,
                          ScriptCompiler::StreamedSource::Encoding encoding) {
+  // It's possible that this will be called with length 0, but don't assume that
+  // the functions this calls handle it gracefully.
+  if (length == 0) return 0;
+
   if (encoding == ScriptCompiler::StreamedSource::UTF8) {
     return v8::internal::Utf8ToUtf16CharacterStream::CopyChars(
         dest, length, src, src_pos, src_length);
   }
 
   unsigned to_fill = length;
   if (to_fill > src_length - *src_pos) to_fill = src_length - *src_pos;
 
   if (encoding == ScriptCompiler::StreamedSource::ONE_BYTE) {
     v8::internal::CopyChars<uint8_t, uint16_t>(dest, src + *src_pos, to_fill);
@@ skipping 343 matching lines @@
       current_data_ = NULL;
       current_data_length_ = 0;
       current_data_offset_ = 0;
     }
   }
   return data_in_buffer;
 }
 
 void ExternalStreamingStream::HandleUtf8SplitCharacters(
     unsigned* data_in_buffer) {
+  // Note the following property of UTF-8 which makes this function possible:
+  // Given any byte, we can always read its local environment (in both
+  // directions) to find out the (possibly multi-byte) character it belongs
+  // to. Single byte characters are of the form 0b0XXXXXXX. The first byte of a
+  // multi-byte character is of the form 0b110XXXXX, 0b1110XXXX or
+  // 0b11110XXX. The continuation bytes are of the form 0b10XXXXXX.
+
   // First check if we have leftover data from the last chunk.
   unibrow::uchar c;
   if (utf8_split_char_buffer_length_ > 0) {
     // Move the bytes which are part of the split character (which started in
-    // the previous chunk) into utf8_split_char_buffer_.
+    // the previous chunk) into utf8_split_char_buffer_. Note that the
+    // continuation bytes are of the form 0b10XXXXXX, thus c >> 6 == 2.
     while (current_data_offset_ < current_data_length_ &&
            utf8_split_char_buffer_length_ < 4 &&
-           (c = current_data_[current_data_offset_]) >
-               unibrow::Utf8::kMaxOneByteChar) {
+           (c = current_data_[current_data_offset_]) >> 6 == 2) {
       utf8_split_char_buffer_[utf8_split_char_buffer_length_] = c;
       ++utf8_split_char_buffer_length_;
       ++current_data_offset_;
     }
 
     // Convert the data in utf8_split_char_buffer_.
     unsigned new_offset = 0;
     unsigned new_chars_in_buffer =
         CopyCharsHelper(buffer_ + *data_in_buffer,
                         kBufferSize - *data_in_buffer, utf8_split_char_buffer_,
@@ skipping 45 matching lines @@
         int end_position)
     : Utf16CharacterStream(),
       source_(data),
       raw_data_(data->GetTwoByteData(start_position)) {
   buffer_cursor_ = raw_data_,
   buffer_end_ = raw_data_ + (end_position - start_position);
   pos_ = start_position;
 }
 
 } }  // namespace v8::internal