IsStringUTF8 unittest and enforcing UTF-8 in JSON deserialization

base/string_util.cc

 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/string_util.h"
 
 #include <ctype.h>
 #include <errno.h>
 #include <math.h>
 #include <stdarg.h>
@@ skipping 499 matching lines @@
 //   the Initial Developer. All Rights Reserved.
 //
 //   Contributor(s):
 //     Scott Collins <scc@mozilla.org> (original author)
 //
 // This is a template so that it can be run on wide and 8-bit strings. We want
 // to run it on wide strings when we have input that we think may have
 // originally been UTF-8, but has been converted to wide characters because
 // that's what we (and Windows) use internally.
 template<typename CHAR>
-static bool IsStringUTF8T(const CHAR* str) {
+static bool IsStringUTF8T(const CHAR* str, int length) {
   bool overlong = false;
   bool surrogate = false;
   bool nonchar = false;
 
   // overlong byte upper bound
   typename ToUnsigned<CHAR>::Unsigned olupper = 0;
 
   // surrogate byte lower bound
   typename ToUnsigned<CHAR>::Unsigned slower = 0;
 
   // incremented when inside a multi-byte char to indicate how many bytes
   // are left in the sequence
   int positions_left = 0;
 
-  for (int i = 0; str[i] != 0; i++) {
+  for (int i = 0; i < length; i++) {
     // This whole function assume an unsigned value so force its conversion to
     // an unsigned value.
     typename ToUnsigned<CHAR>::Unsigned c = str[i];
     if (c < 0x80)
       continue;  // ASCII
 
     if (c <= 0xC1) {
       // [80-BF] where not expected, [C0-C1] for overlong
       return false;
     } else if (IsBegin2ByteUTF8(c)) {
       positions_left = 1;
     } else if (IsBegin3ByteUTF8(c)) {
       positions_left = 2;
       if (c == 0xE0) {
         // to exclude E0[80-9F][80-BF]
         overlong = true;
         olupper = 0x9F;
       } else if (c == 0xED) {
         // ED[A0-BF][80-BF]: surrogate codepoint
         surrogate = true;
         slower = 0xA0;
       } else if (c == 0xEF) {
         // EF BF [BE-BF] : non-character
+        // TODO(jungshik): EF B7 [90-AF] should be checked as well.
         nonchar = true;
       }
     } else if (c <= 0xF4) {
       positions_left = 3;
       nonchar = true;
       if (c == 0xF0) {
         // to exclude F0[80-8F][80-BF]{2}
         overlong = true;
         olupper = 0x8F;
       } else if (c == 0xF4) {
@@ skipping 23 matching lines @@
       if (!IsInUTF8Sequence(c) || (overlong && c <= olupper) ||
           (surrogate && slower <= c) || (nonchar && !positions_left) ) {
         return false;
       }
       overlong = surrogate = false;
     }
   }
   return true;
 }
 
-bool IsStringUTF8(const char* str) {
-  return IsStringUTF8T(str);
+bool IsStringUTF8(const std::string& str) {
+  return IsStringUTF8T(str.data(), str.length());
 }
 
-bool IsStringWideUTF8(const wchar_t* str) {
-  return IsStringUTF8T(str);
+bool IsStringWideUTF8(const std::wstring& str) {
+  return IsStringUTF8T(str.data(), str.length());
 }
 
 template<typename Iter>
 static inline bool DoLowerCaseEqualsASCII(Iter a_begin,
                                           Iter a_end,
                                           const char* b) {
   for (Iter it = a_begin; it != a_end; ++it, ++b) {
     if (!*b || ToLowerASCII(*it) != *b)
       return false;
   }
@@ skipping 808 matching lines @@
 
 }  // namespace
 
 size_t base::strlcpy(char* dst, const char* src, size_t dst_size) {
   return lcpyT<char>(dst, src, dst_size);
 }
 size_t base::wcslcpy(wchar_t* dst, const wchar_t* src, size_t dst_size) {
   return lcpyT<wchar_t>(dst, src, dst_size);
 }