UTF-8 detector for pages missing encoding info

third_party/WebKit/Source/platform/text/TextEncodingDetector.cpp

Index: third_party/WebKit/Source/platform/text/TextEncodingDetector.cpp
diff --git a/third_party/WebKit/Source/platform/text/TextEncodingDetector.cpp b/third_party/WebKit/Source/platform/text/TextEncodingDetector.cpp
index 616aac0114d74ae3f284b97a740124242c7c7e3e..a03eb0e9f957fefee468b677b98989c3f738269b 100644
--- a/third_party/WebKit/Source/platform/text/TextEncodingDetector.cpp
+++ b/third_party/WebKit/Source/platform/text/TextEncodingDetector.cpp
@@ -33,6 +33,7 @@
 #include "wtf/text/TextEncoding.h"
 #include <unicode/ucnv.h>
 #include <unicode/ucsdet.h>
+#include <unicode/utf8.h>
 
 namespace blink {
 
@@ -112,4 +113,21 @@ bool detectTextEncoding(const char* data, size_t length,
     return false;
 }
 
+bool isUTF8Encoded(const char* data, size_t length)
+{
+    int32_t srcLen = static_cast<int32_t>(length);
+    int32_t charIndex = 0;
+    bool markDetected = false;
+
+    while (charIndex < srcLen) {
+        int32_t codePoint;
+        if ((uint8_t)(data[charIndex]) >= 0x80)
+            markDetected = true;
+        U8_NEXT(data, charIndex, srcLen, codePoint);
+        if (!U_IS_UNICODE_CHAR(codePoint))

[aelias_OOO_until_Jul13, 2016/02/24 04:37:54]

According to http://icu-project.org/apiref/icu4c/utf8_8h.html reference for
U8_NEXT,  "If the offset points to a trail byte or an illegal UTF-8 sequence,
then c is set to a negative value.", so how about just checking for < 0 instead?

[Jinsuk Kim, 2016/02/24 06:54:54]

Thanks for looking into the detail. Ran the unittests again with codePoint < 0
to find that it works in almost all cases but for following:

    // Non-characters : U+xxFFF[EF] where xx is 0x00 through 0x10 and
<FDD0,FDEF>
    EXPECT_FALSE(isUTF8Encoded("\xef\xbf\xbe", 3)); // U+FFFE

    EXPECT_FALSE(isUTF8Encoded("\xf3\xbf\xbf\xbf", 4)); // U+10FFFF
    EXPECT_FALSE(isUTF8Encoded("\xef\xb7\x90", 3)); // U+FDD0
    EXPECT_FALSE(isUTF8Encoded("\xef\xb7\xaf", 3)); // U+FDEF

Not sure how important/practical these cases are.

+            return false;
+    }
+    return markDetected;
+}
+
 } // namespace blink