UTF-8 detector for pages missing encoding info

third_party/WebKit/Source/platform/text/TextEncodingDetector.cpp

 /*
  * Copyright (C) 2008, 2009 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 15 matching lines @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "platform/text/TextEncodingDetector.h"
 
 #include "wtf/text/TextEncoding.h"
 #include <unicode/ucnv.h>
 #include <unicode/ucsdet.h>
+#include <unicode/utf8.h>
 
 namespace blink {
 
 bool detectTextEncoding(const char* data, size_t length,
     const char* hintEncodingName, WTF::TextEncoding* detectedEncoding)
 {
     *detectedEncoding = WTF::TextEncoding();
     int matchesCount = 0;
     UErrorCode status = U_ZERO_ERROR;
     UCharsetDetector* detector = ucsdet_open(&status);
@@ skipping 59 matching lines @@
         encoding = ucsdet_getName(matches[0], &status);
     if (U_SUCCESS(status)) {
         *detectedEncoding = WTF::TextEncoding(encoding);
         ucsdet_close(detector);
         return true;
     }
     ucsdet_close(detector);
     return false;
 }
 
+bool isUTF8Encoded(const char* data, size_t length)
+{
+    int32_t srcLen = static_cast<int32_t>(length);
+    int32_t charIndex = 0;
+    bool markDetected = false;
+
+    while (charIndex < srcLen) {
+        int32_t codePoint;
+        if ((uint8_t)(data[charIndex]) >= 0x80)
+            markDetected = true;
+        U8_NEXT(data, charIndex, srcLen, codePoint);
+        if (!U_IS_UNICODE_CHAR(codePoint))

[aelias_OOO_until_Jul13, 2016/02/24 04:37:54]

According to http://icu-project.org/apiref/icu4c/utf8_8h.html reference for
U8_NEXT,  "If the offset points to a trail byte or an illegal UTF-8 sequence,
then c is set to a negative value.", so how about just checking for < 0 instead?

[Jinsuk Kim, 2016/02/24 06:54:54]

Thanks for looking into the detail. Ran the unittests again with codePoint < 0
to find that it works in almost all cases but for following:

    // Non-characters : U+xxFFF[EF] where xx is 0x00 through 0x10 and
<FDD0,FDEF>
    EXPECT_FALSE(isUTF8Encoded("\xef\xbf\xbe", 3)); // U+FFFE

    EXPECT_FALSE(isUTF8Encoded("\xf3\xbf\xbf\xbf", 4)); // U+10FFFF
    EXPECT_FALSE(isUTF8Encoded("\xef\xb7\x90", 3)); // U+FDD0
    EXPECT_FALSE(isUTF8Encoded("\xef\xb7\xaf", 3)); // U+FDEF

Not sure how important/practical these cases are.

+            return false;
+    }
+    return markDetected;
+}
+
 } // namespace blink