Fix bad-char handling in utf-8 streaming streams. Also add test.

src/unicode.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // This file was generated at 2014-10-08 15:25:47.940335
 
 #include "src/unicode.h"
 #include "src/unicode-inl.h"
 #include <stdio.h>
 #include <stdlib.h>
@@ skipping 315 matching lines @@
       // Store the kind - 1 (i.e., remaining bytes) in the top byte, value
       // in the bottom three.
       *buffer = (kind - 1) << 24 | (next & mask);
       return kIncomplete;
     } else {
       // No buffer, and not the start of a 1-byte char (handled at the
       // beginning), and not the start of a 2..4 byte char? Bad char.
       *buffer = 0;
       return kBadChar;
     }
+  } else if (*buffer <= 0xff) {
+    // We have one unprocessed byte left (from the last else case in this if
+    // statement).
+    uchar previous = *buffer;
+    *buffer = 0;
+    uchar t = ValueOfIncremental(previous, buffer);
+    if (t == kIncomplete) {
+      // If we have an incomplete character, process both the previous and the
+      // next byte at once.
+      return ValueOfIncremental(next, buffer);
+    } else {
+      // Otherwise, process the previous byte and save the next byte for next
+      // time.
+      DCHECK_EQ(0, *buffer);
+      *buffer = next;
+      return t;
+    }
+  } else if (IsContinuationCharacter(next)) {
+    // We're inside of a character, as described by buffer.
+
+    // How many bytes (excluding this one) do we still expect?
+    uint8_t count = (*buffer >> 24) - 1;
+    // Update the value.
+    uint32_t value = ((*buffer & 0xffffff) << 6) | (next & 0x3F);
+    if (count) {
+      *buffer = count << 24 | value;
+      return kIncomplete;
+    } else {
+      *buffer = 0;
+      return value;
+    }
   } else {
-    // We're inside of a character, as described by buffer.
-    if (IsContinuationCharacter(next)) {
-      // How many bytes (excluding this one) do we still expect?
-      uint8_t count = (*buffer >> 24) - 1;
-      // Update the value.
-      uint32_t value = ((*buffer & 0xffffff) << 6) | (next & 0x3F);
-      if (count) {
-        *buffer = count << 24 | value;
-        return kIncomplete;
-      } else {
-        *buffer = 0;
-        return value;
-      }
-    } else {
-      // Within a character, but not a continuation character? Bad char.
-      *buffer = 0;
-      return kBadChar;
-    }
+    // Within a character, but not a continuation character? Then the
+    // previous char was a bad char. But we need to save the current
+    // one.
+    *buffer = next;
+    return kBadChar;
   }
 }
 
+uchar Utf8::ValueOfIncrementalFinish(Utf8IncrementalBuffer* buffer) {
+  DCHECK_NOT_NULL(buffer);
+  if (*buffer == 0) {
+    return kBufferEmpty;
+  } else {
+    // Process left-over chars. An incomplete char at the end maps to kBadChar.
+    uchar t = ValueOfIncremental(0, buffer);
+    return (t == kIncomplete) ? kBadChar : t;
+  }
+}
+
 bool Utf8::Validate(const byte* bytes, size_t length) {
   size_t cursor = 0;
 
   // Performance optimization: Skip over single-byte values first.
   while (cursor < length && bytes[cursor] <= kMaxOneByteChar) {
     ++cursor;
   }
 
   while (cursor < length) {
     uchar c = ValueOf(bytes + cursor, length - cursor, &cursor);
@@ skipping 3129 matching lines @@
              sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
          +
          kCanonicalizationRangeMultiStrings1Size *
              sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
          +
          kCanonicalizationRangeMultiStrings7Size *
              sizeof(MultiCharacterSpecialCase<1>);  // NOLINT
 }
 
 }  // namespace unibrow