Rework scanner-character-streams.

src/unicode.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // This file was generated at 2014-10-08 15:25:47.940335
 
 #include "src/unicode.h"
 #include "src/unicode-inl.h"
 #include <stdio.h>
 #include <stdlib.h>
@@ skipping 172 matching lines @@
         default:
           return 0;
       }
       return -1;
     }
   } else {
     return 0;
   }
 }
 
-
-static inline size_t NonASCIISequenceLength(byte first) {
+static inline uint8_t NonASCIISequenceLength(byte first) {
   // clang-format off
   static const uint8_t lengths[256] = {
       // The first 128 entries correspond to ASCII characters.
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ skipping 93 matching lines @@
   }
   if (!IsContinuationCharacter(str[3])) {
     *cursor += 1;
     return kBadChar;
   }
   *cursor += 4;
   return ((str[0] << 18) + (str[1] << 12) + (str[2] << 6) + str[3]) -
          0x03C82080;
 }
 
+uchar Utf8::ValueOfIncremental(byte next, Utf8IncrementalBuffer& buffer) {
+  // The common case: 1-byte Utf8 (and no incomplete char in the buffer)
+  if (V8_LIKELY(next <= kMaxOneByteChar && buffer == 0)) {
+    return static_cast<uchar>(next);
+  }
+
+  // All other cases:

[marja, 2016/09/07 09:17:57]

Why does it make sense to do NonAsciiSequenceLength(next) if buffer is not 0,
that is, next is a continuation character?

I'd reorder this code like this:

fast path (which you have now)

if (buffer == 0) {
  this is the start of a multi-byte character
  NonAsciiSequenceLength(next)
  etc
} else {
  expect a continuation character, update the buffer based on that (otherwise
kBadChar)
}

[vogelheim, 2016/09/07 12:32:56]

Done. Your version makes much more sense.

+  uint32_t kind = NonASCIISequenceLength(next);
+  switch (kind) {
+    case 0:
+      if (IsContinuationCharacter(next)) {
+        // How many bytes (excluding this one) do we still expect?
+        uint8_t count = (buffer >> 24) - 1;
+        // Update the value.
+        uint32_t value = ((buffer & 0xffffff) << 6) | (next & 0x3F);
+        if (count) {
+          buffer = count << 24 | value;
+          return kIncomplete;
+        } else {
+          buffer = 0;
+          return value;
+        }
+      } else {
+        // Not a continuation character? Bad char.
+        buffer = 0;
+        return kBadChar;
+      }
+    case 2:
+    case 3:
+    case 4:
+      if (buffer == 0) {
+        // Start of 2..4 byte character, and no buffer.
+
+        // The mask for the lower bits depends on the kind, and is
+        // 0x1F, 0x0F, 0x07 for kinds 2, 3, 4 respectively. We can get that
+        // with one shift.
+        uint8_t mask = 0x7f >> kind;
+
+        // Store the kind - 1 (i.e., remaining bytes) in the top byte, value
+        // in the bottom three.
+        buffer = (kind - 1) << 24 | (next & mask);
+        return kIncomplete;
+      } else {
+        // Start of new character, but we still have a buffer? Bad char.
+        buffer = 0;
+        return kBadChar;
+      }
+    default:
+      UNREACHABLE();
+      return kBadChar;
+  }
+}
+
 bool Utf8::Validate(const byte* bytes, size_t length) {
   size_t cursor = 0;
 
   // Performance optimization: Skip over single-byte values first.
   while (cursor < length && bytes[cursor] <= kMaxOneByteChar) {
     ++cursor;
   }
 
   while (cursor < length) {
     uchar c = ValueOf(bytes + cursor, length - cursor, &cursor);
@@ skipping 3129 matching lines @@
              sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
          +
          kCanonicalizationRangeMultiStrings1Size *
              sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
          +
          kCanonicalizationRangeMultiStrings7Size *
              sizeof(MultiCharacterSpecialCase<1>);  // NOLINT
 }
 
 }  // namespace unibrow