Rework scanner-character-streams.

src/unicode.cc

Index: src/unicode.cc
diff --git a/src/unicode.cc b/src/unicode.cc
index db98be867579740b0e599c84c5b8fc164d834665..d82c8163f43813ae200387452d12c93f824ccf2e 100644
--- a/src/unicode.cc
+++ b/src/unicode.cc
@@ -190,8 +190,7 @@ static int LookupMapping(const int32_t* table,
   }
 }
 
-
-static inline size_t NonASCIISequenceLength(byte first) {
+static inline uint8_t NonASCIISequenceLength(byte first) {
   // clang-format off
   static const uint8_t lengths[256] = {
       // The first 128 entries correspond to ASCII characters.
@@ -305,6 +304,59 @@ uchar Utf8::CalculateValue(const byte* str, size_t max_length, size_t* cursor) {
          0x03C82080;
 }
 
+uchar Utf8::ValueOfIncremental(byte next, Utf8IncrementalBuffer& buffer) {
+  // The common case: 1-byte Utf8 (and no incomplete char in the buffer)
+  if (V8_LIKELY(next <= kMaxOneByteChar && buffer == 0)) {
+    return static_cast<uchar>(next);
+  }
+
+  // All other cases:

[marja, 2016/09/07 09:17:57]

Why does it make sense to do NonAsciiSequenceLength(next) if buffer is not 0,
that is, next is a continuation character?

I'd reorder this code like this:

fast path (which you have now)

if (buffer == 0) {
  this is the start of a multi-byte character
  NonAsciiSequenceLength(next)
  etc
} else {
  expect a continuation character, update the buffer based on that (otherwise
kBadChar)
}

[vogelheim, 2016/09/07 12:32:56]

Done. Your version makes much more sense.

+  uint32_t kind = NonASCIISequenceLength(next);
+  switch (kind) {
+    case 0:
+      if (IsContinuationCharacter(next)) {
+        // How many bytes (excluding this one) do we still expect?
+        uint8_t count = (buffer >> 24) - 1;
+        // Update the value.
+        uint32_t value = ((buffer & 0xffffff) << 6) | (next & 0x3F);
+        if (count) {
+          buffer = count << 24 | value;
+          return kIncomplete;
+        } else {
+          buffer = 0;
+          return value;
+        }
+      } else {
+        // Not a continuation character? Bad char.
+        buffer = 0;
+        return kBadChar;
+      }
+    case 2:
+    case 3:
+    case 4:
+      if (buffer == 0) {
+        // Start of 2..4 byte character, and no buffer.
+
+        // The mask for the lower bits depends on the kind, and is
+        // 0x1F, 0x0F, 0x07 for kinds 2, 3, 4 respectively. We can get that
+        // with one shift.
+        uint8_t mask = 0x7f >> kind;
+
+        // Store the kind - 1 (i.e., remaining bytes) in the top byte, value
+        // in the bottom three.
+        buffer = (kind - 1) << 24 | (next & mask);
+        return kIncomplete;
+      } else {
+        // Start of new character, but we still have a buffer? Bad char.
+        buffer = 0;
+        return kBadChar;
+      }
+    default:
+      UNREACHABLE();
+      return kBadChar;
+  }
+}
+
 bool Utf8::Validate(const byte* bytes, size_t length) {
   size_t cursor = 0;