Return one U+fffd for longest subpart of incomplete utf-8 character.

third_party/WebKit/Source/wtf/text/TextCodecUTF8.cpp

 /*
  * Copyright (C) 2004, 2006, 2008, 2011 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 18 matching lines @@
 #include "wtf/text/CString.h"
 #include "wtf/text/CharacterNames.h"
 #include "wtf/text/StringBuffer.h"
 #include "wtf/text/TextCodecASCIIFastPath.h"
 #include <memory>
 
 namespace WTF {
 
 using namespace WTF::Unicode;
 
-const int nonCharacter = -1;
+// We'll use nonCharacter* constants to signal invalid utf-8.
+// The number in the name signals how many input bytes were invalid.
+const int nonCharacter1 = -1;
+const int nonCharacter2 = -2;
+const int nonCharacter3 = -3;
+
+bool isNonCharacter(int character) {
+  return character >= nonCharacter3 && character <= nonCharacter1;
+}
 
 std::unique_ptr<TextCodec> TextCodecUTF8::create(const TextEncoding&,
                                                  const void*) {
   return wrapUnique(new TextCodecUTF8);
 }
 
 void TextCodecUTF8::registerEncodingNames(EncodingNameRegistrar registrar) {
   registrar("UTF-8", "UTF-8");
 
   // Additional aliases that originally were present in the encoding
@@ skipping 30 matching lines @@
       4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
   return lengths[firstByte];
 }
 
 static inline int decodeNonASCIISequence(const uint8_t* sequence,
                                          unsigned length) {
   ASSERT(!isASCII(sequence[0]));
   if (length == 2) {
     ASSERT(sequence[0] <= 0xDF);
     if (sequence[0] < 0xC2)
-      return nonCharacter;
+      return nonCharacter1;
     if (sequence[1] < 0x80 || sequence[1] > 0xBF)
-      return nonCharacter;
+      return nonCharacter1;
     return ((sequence[0] << 6) + sequence[1]) - 0x00003080;
   }
   if (length == 3) {
     ASSERT(sequence[0] >= 0xE0 && sequence[0] <= 0xEF);
     switch (sequence[0]) {
       case 0xE0:
         if (sequence[1] < 0xA0 || sequence[1] > 0xBF)
-          return nonCharacter;
+          return nonCharacter1;
         break;
       case 0xED:
         if (sequence[1] < 0x80 || sequence[1] > 0x9F)
-          return nonCharacter;
+          return nonCharacter1;
         break;
       default:
         if (sequence[1] < 0x80 || sequence[1] > 0xBF)
-          return nonCharacter;
+          return nonCharacter1;
     }
     if (sequence[2] < 0x80 || sequence[2] > 0xBF)
-      return nonCharacter;
+      return nonCharacter2;
     return ((sequence[0] << 12) + (sequence[1] << 6) + sequence[2]) -
            0x000E2080;
   }
   ASSERT(length == 4);
   ASSERT(sequence[0] >= 0xF0 && sequence[0] <= 0xF4);
   switch (sequence[0]) {
     case 0xF0:
       if (sequence[1] < 0x90 || sequence[1] > 0xBF)
-        return nonCharacter;
+        return nonCharacter1;
       break;
     case 0xF4:
       if (sequence[1] < 0x80 || sequence[1] > 0x8F)
-        return nonCharacter;
+        return nonCharacter1;
       break;
     default:
       if (sequence[1] < 0x80 || sequence[1] > 0xBF)
-        return nonCharacter;
+        return nonCharacter1;
   }
   if (sequence[2] < 0x80 || sequence[2] > 0xBF)
-    return nonCharacter;
+    return nonCharacter2;
   if (sequence[3] < 0x80 || sequence[3] > 0xBF)
-    return nonCharacter;
+    return nonCharacter3;
   return ((sequence[0] << 18) + (sequence[1] << 12) + (sequence[2] << 6) +
           sequence[3]) -
          0x03C82080;
 }
 
 static inline UChar* appendCharacter(UChar* destination, int character) {
-  ASSERT(character != nonCharacter);
-  ASSERT(!U_IS_SURROGATE(character));
+  DCHECK(!isNonCharacter(character));
+  DCHECK(!U_IS_SURROGATE(character));
   if (U_IS_BMP(character)) {
     *destination++ = static_cast<UChar>(character);
   } else {
     *destination++ = U16_LEAD(character);
     *destination++ = U16_TRAIL(character);
   }
   return destination;
 }
 
 void TextCodecUTF8::consumePartialSequenceByte() {
@@ skipping 97 matching lines @@
         if (stopOnError)
           return false;
         continue;
       }
       memcpy(m_partialSequence + m_partialSequenceSize, source,
              count - m_partialSequenceSize);
       source += count - m_partialSequenceSize;
       m_partialSequenceSize = count;
     }
     int character = decodeNonASCIISequence(m_partialSequence, count);
-    if (character == nonCharacter) {
+    if (isNonCharacter(character)) {
       handleError(destination, stopOnError, sawError);
+      count = -character;

[marja, 2016/11/16 08:41:38]

Looks like count doesn't affect anything

[vogelheim, 2016/11/16 09:53:16]

Removed.

       if (stopOnError)
         return false;
       continue;
     }
 
     m_partialSequenceSize -= count;
     destination = appendCharacter(destination, character);
   } while (m_partialSequenceSize);
 
   return false;
@@ skipping 50 matching lines @@
             break;
           if (!isASCII(*source))
             continue;
         }
         *destination++ = *source++;
         continue;
       }
       int count = nonASCIISequenceLength(*source);
       int character;
       if (count == 0) {
-        character = nonCharacter;
+        character = nonCharacter1;
       } else {
         if (count > end - source) {
           SECURITY_DCHECK(end - source <
                           static_cast<ptrdiff_t>(sizeof(m_partialSequence)));
           ASSERT(!m_partialSequenceSize);
           m_partialSequenceSize = end - source;
           memcpy(m_partialSequence, source, m_partialSequenceSize);
           source = end;
           break;
         }
         character = decodeNonASCIISequence(source, count);
       }
-      if (character == nonCharacter) {
+      if (isNonCharacter(character)) {
+        count = -character;

[marja, 2016/11/16 08:41:38]

Ditto

[vogelheim, 2016/11/16 09:53:16]

Removed. (All actual processing of non-ASCII characters is done in
upConvertTo16Bit.)

         sawError = true;
         if (stopOnError)
           break;
 
         goto upConvertTo16Bit;
       }
       if (character > 0xff)
         goto upConvertTo16Bit;
 
       source += count;
@@ skipping 47 matching lines @@
             break;
           if (!isASCII(*source))
             continue;
         }
         *destination16++ = *source++;
         continue;
       }
       int count = nonASCIISequenceLength(*source);
       int character;
       if (count == 0) {
-        character = nonCharacter;
+        character = nonCharacter1;
       } else {
         if (count > end - source) {
           SECURITY_DCHECK(end - source <
                           static_cast<ptrdiff_t>(sizeof(m_partialSequence)));
           ASSERT(!m_partialSequenceSize);
           m_partialSequenceSize = end - source;
           memcpy(m_partialSequence, source, m_partialSequenceSize);
           source = end;
           break;
         }
         character = decodeNonASCIISequence(source, count);
       }
-      if (character == nonCharacter) {
+      if (isNonCharacter(character)) {
         sawError = true;
         if (stopOnError)
           break;
         // Each error generates a replacement character and consumes one byte.
         *destination16++ = replacementCharacter;
-        ++source;
+        source -= character;
         continue;
       }
       source += count;
       destination16 = appendCharacter(destination16, character);
     }
   } while (flush && m_partialSequenceSize);
 
   buffer16.shrink(destination16 - buffer16.characters());
 
   return String::adopt(buffer16);
@@ skipping 32 matching lines @@
   return encodeCommon(characters, length);
 }
 
 CString TextCodecUTF8::encode(const LChar* characters,
                               size_t length,
                               UnencodableHandling) {
   return encodeCommon(characters, length);
 }
 
 }  // namespace WTF