Rework scanner-character-streams.

src/unicode.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_UNICODE_H_
 #define V8_UNICODE_H_
 
 #include <sys/types.h>
 #include "src/globals.h"
 #include "src/utils.h"
@@ skipping 123 matching lines @@
   static inline unsigned EncodeOneByte(char* out, uint8_t c);
   static inline unsigned Encode(char* out,
                                 uchar c,
                                 int previous,
                                 bool replace_invalid = false);
   static uchar CalculateValue(const byte* str, size_t length, size_t* cursor);
 
   // The unicode replacement character, used to signal invalid unicode
   // sequences (e.g. an orphan surrogate) when converting to a UTF-8 encoding.
   static const uchar kBadChar = 0xFFFD;
+  static const uchar kBufferEmpty = 0x0;
+  static const uchar kIncomplete = 0xFFFFFFFC;  // any non-valid code point.
   static const unsigned kMaxEncodedSize   = 4;
   static const unsigned kMaxOneByteChar   = 0x7f;
   static const unsigned kMaxTwoByteChar   = 0x7ff;
   static const unsigned kMaxThreeByteChar = 0xffff;
   static const unsigned kMaxFourByteChar  = 0x1fffff;
 
   // A single surrogate is coded as a 3 byte UTF-8 sequence, but two together
   // that match are coded as a 4 byte UTF-8 sequence.
   static const unsigned kBytesSavedByCombiningSurrogates = 2;
   static const unsigned kSizeOfUnmatchedSurrogate = 3;
   // The maximum size a single UTF-16 code unit may take up when encoded as
   // UTF-8.
   static const unsigned kMax16BitCodeUnitSize  = 3;
   static inline uchar ValueOf(const byte* str, size_t length, size_t* cursor);
 
+  typedef uint32_t Utf8IncrementalBuffer;
+  static uchar ValueOfIncremental(byte next_byte,
+                                  Utf8IncrementalBuffer& buffer);
+
   // Excludes non-characters from the set of valid code points.
   static inline bool IsValidCharacter(uchar c);
 
   static bool Validate(const byte* str, size_t length);
 };
 
 struct Uppercase {
   static bool Is(uchar c);
 };
 struct Lowercase {
@@ skipping 48 matching lines @@
   static const int kMaxWidth = 1;
   static int Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr);
 };
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_H_