Rework scanner-character-streams.

src/unicode.h

Index: src/unicode.h
diff --git a/src/unicode.h b/src/unicode.h
index 35717bca868da4e17bff98ee7012932c7149b599..bbe4209f6815e8559f55fb0f972d4ba7b8817610 100644
--- a/src/unicode.h
+++ b/src/unicode.h
@@ -141,6 +141,8 @@ class Utf8 {
   // The unicode replacement character, used to signal invalid unicode
   // sequences (e.g. an orphan surrogate) when converting to a UTF-8 encoding.
   static const uchar kBadChar = 0xFFFD;
+  static const uchar kBufferEmpty = 0x0;
+  static const uchar kIncomplete = 0xFFFFFFFC;  // any non-valid code point.
   static const unsigned kMaxEncodedSize   = 4;
   static const unsigned kMaxOneByteChar   = 0x7f;
   static const unsigned kMaxTwoByteChar   = 0x7ff;
@@ -156,6 +158,10 @@ class Utf8 {
   static const unsigned kMax16BitCodeUnitSize  = 3;
   static inline uchar ValueOf(const byte* str, size_t length, size_t* cursor);
 
+  typedef uint32_t Utf8IncrementalBuffer;
+  static uchar ValueOfIncremental(byte next_byte,
+                                  Utf8IncrementalBuffer& buffer);

[nickie, 2016/09/07 13:48:44]

I think this reference should be turned into a pointer, according to the code
style.  I'm not sure if it's enforced by the CQ.

[vogelheim, 2016/09/08 15:02:18]

Done.

+
   // Excludes non-characters from the set of valid code points.
   static inline bool IsValidCharacter(uchar c);