CL for perf tryjob on linux

third_party/cld/encodings/compact_enc_det/compact_enc_det_hint_code.cc

Index: third_party/cld/encodings/compact_enc_det/compact_enc_det_hint_code.cc
diff --git a/third_party/cld/encodings/compact_enc_det/compact_enc_det_hint_code.cc b/third_party/cld/encodings/compact_enc_det/compact_enc_det_hint_code.cc
new file mode 100644
index 0000000000000000000000000000000000000000..938563573e4ca862e8e90a8af1cd6296ac671d4e
--- /dev/null
+++ b/third_party/cld/encodings/compact_enc_det/compact_enc_det_hint_code.cc
@@ -0,0 +1,430 @@
+//
+// Copyright 2006 Google Inc.  All Rights Reserved.
+// Author: dsites@google.com (Dick Sites)
+//
+
+
+#include "encodings/compact_enc_det/compact_enc_det_hint_code.h"
+
+#include <ctype.h>                      // for isalpha
+#include <string.h>                     // for NULL, memchr, strlen, etc
+
+#include "base/basictypes.h"            // for uint8, uint32
+//#include "webutil/url/url.h"            // for URL
+
+// Upper to lower, keep digits, everything else to minus '-' (2d)
+static const char kCharsetToLowerTbl[256] = {
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37, 0x38,0x39,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+};
+
+
+static const char kIsAlpha[256] = {
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+  0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char kIsDigit[256] = {
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 1,1,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char* kFakeEncodingName[] = {
+  "FakeEnc100", "FakeEnc101", "FakeEnc102", "FakeEnc103", "FakeEnc104",
+  "FakeEnc105", "FakeEnc106", "FakeEnc107", "FakeEnc108", "FakeEnc109",
+  "FakeEnc110", "FakeEnc111", "FakeEnc112", "FakeEnc113", "FakeEnc114",
+  "FakeEnc115", "FakeEnc116", "FakeEnc117", "FakeEnc118", "FakeEnc119",
+};
+static const char* kFakeEncodingName2[] = {
+  "FakeEnc_0", "FakeEnc_1", "FakeEnc_2", "FakeEnc_3", "FakeEnc_4",
+};
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc) {
+  if (enc < 0) {
+    return "~";
+  }
+  if (enc == ISO_8859_1) {
+    return "Latin1";   // I can't stand "ASCII" for this
+  }
+  if (enc < NUM_ENCODINGS) {
+    return EncodingName(enc);
+  }
+  // allow fake names, for exploration
+  if ((NUM_ENCODINGS <= enc) && (enc < (NUM_ENCODINGS + 4))) {
+    return kFakeEncodingName2[enc - NUM_ENCODINGS];
+  }
+  if ((100 <= enc) && (enc < 120)) {
+    return kFakeEncodingName[enc - 100];
+  }
+  return "~";
+}
+
+
+// http://www.iana.org/assignments/character-sets says charset name is up to
+// 40 bytes of any printable ASCII, but that can't be right
+// when parsing HTML; at least quote is not allowed. The list
+// here includes all punctuation in all registered names as of April 2006
+static const char* kWordLetters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                                  "abcdefghijklmnopqrstuvwxyz"
+                                  "0123456789"
+                                  "-_.:()";
+
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str) {
+  string res("________");     // eight underscores
+  int l_ptr = 0;
+  int d_ptr = 0;
+  for (int i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc]) {
+      if (l_ptr < 4) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    } else if (kIsDigit[uc]) {
+      if (d_ptr < 4) {
+        res[4 + d_ptr] = kCharsetToLowerTbl[uc];
+      } else {
+        // Keep last 4 digits by shifting left
+        res[4] = res[5];
+        res[5] = res[6];
+        res[6] = res[7];
+        res[7] = kCharsetToLowerTbl[uc];
+      }
+      d_ptr++;
+    }   // If neither letter nor digit, drop entirely
+  }
+  return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str) {
+  string res("____");     // four underscores
+  int l_ptr = 0;
+  for (int i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc] | kIsDigit[uc]) {
+      if (l_ptr < 4) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    }
+  }
+  return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar8(const string& str) {
+  string res("________");     // eight dots
+  int l_ptr = 0;
+  for (int i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc] | kIsDigit[uc]) {
+      if (l_ptr < 8) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    }
+  }
+  return res;
+}
+
+// A-Z to a-z and all non-digits-letters to minus '-'
+void StringToLowercase(string* str) {
+  for (int i = 0; i < str->size(); i++) {
+    (*str)[i] = kCharsetToLowerTbl[static_cast<uint8>((*str)[i])];
+  }
+}
+
+bool AllDigits(const string& str, int wordstart_offset, int len) {
+  for (int i = 0; i < len; i++) {
+    char c = str[wordstart_offset + i];
+    if ('9' < c) {return false;}
+    if (c < '0') {return false;}
+  }
+  return true;
+}
+
+
+inline char lower(char c) { return c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c; }
+inline char upper(char c) { return c >= 'a' && c <= 'z' ? c - 'a' + 'A' : c; }
+
+// This is like strstr, but without assuming that the char* is null-terminated,
+// and the comparison is case-insensitive. (Real files have "Meta", "meta",
+// and "META". Some files have more than one version. Likewise for "charset".)
+const char* FindSubstring(const char* start,
+                          const char* end,
+                          const string& substring) {
+  const char* const sub = substring.data();
+  const int sublen = substring.size();
+  const char first_lower = lower(*sub);
+  const char first_upper = upper(*sub);
+  const char* const rest = sub + 1;
+  const int rest_len = sublen - 1;
+  end -= sublen;
+
+  while (start < end) {
+    // Find the first occurrence of the lowercase version of the first
+    // letter of the substring.
+    const char* p =
+      static_cast<const char*>(memchr(start, first_lower, end - start));
+    if (first_lower != first_upper) {
+      // If that's a letter, look for the uppercase version, too.
+      const char* q =
+        static_cast<const char*>(memchr(start, first_upper, end - start));
+      if (p == NULL ||
+          (q != NULL && q < p)) {
+        p = q;  // Uppercase came first.
+      }
+    }
+    if (p == NULL) return NULL;
+    if (strncasecmp(p + 1, rest, rest_len) == 0) return p;
+    start = p + 1;
+  }
+  return NULL;
+}
+
+size_t FindSubstring(const string& str, const string& substring) {
+  const char* data = str.data();
+  const char* z = FindSubstring(data, data + str.size(), substring);
+  return z == NULL ? string::npos : z - data;
+}
+
+// Get charset value from string
+// Normalize: truncate to 16 chars and make lowercase
+string GetCharset(const string& str) {
+  auto charset_offset = FindSubstring(str, "charset");
+  if (charset_offset == string::npos) {
+    charset_offset = FindSubstring(str, "encoding");
+    if (charset_offset == string::npos) {
+      return "";
+    }
+  }
+  int eq_offset = str.find("=", charset_offset);
+  if (eq_offset == string::npos) {
+    return "";
+  }
+  // skip same-line whitespace and quote after equal
+  int wordstart_offset = str.find_first_not_of(" \t\"\'", eq_offset + 1);
+  if (wordstart_offset == string::npos) {
+    return "";
+  }
+  int len = str.length() - wordstart_offset;
+  int wordend_offset = str.find_first_not_of(kWordLetters, wordstart_offset);
+  if (wordend_offset != string::npos) {
+    len = wordend_offset - wordstart_offset;
+  }
+
+  // If too long, it must be bogus
+  if (18 < len) {
+    return "";
+  }
+  // If <= 1 char, it must be bogus
+  if (len <= 1) {
+    return "";
+  }
+  // If all digits and less than 3 or more than 6 digits, it must be bogus
+  if (AllDigits(str, wordstart_offset, len) && ((len < 3) || (6 < len))) {
+    return "";
+  }
+
+  // Extract and convert to lowercase (converting punct to '-')
+  string charset(str.substr(wordstart_offset, len));
+  StringToLowercase(&charset);
+
+  // Strip common prefixes - x- 3d
+  while ((charset.size() > 2) &&
+         (charset[0] == '3') &&
+         (charset[1] == 'd')) {
+    charset.erase(0, 2);
+  }
+  while ((charset.size() > 2) &&
+         (charset[0] == 'x') &&
+         (charset[1] == '-')) {
+    charset.erase(0, 2);
+  }
+  while ((charset.size() > 1) &&
+         (charset[0] == '-')) {
+    charset.erase(0, 1);
+  }
+
+
+  // Strip common suffixes - -80 -19xx -200x
+  while ((charset.size() > 1) &&
+      (charset[charset.size() - 1] == '-')) {
+    charset.erase(charset.size() - 1, 1);
+  }
+
+  if ((charset.size() > 3) &&
+      (charset[charset.size() - 3] == '-') &&
+      (charset[charset.size() - 2] == '8') &&
+      (charset[charset.size() - 1] == '0')) {
+    charset.erase(charset.size() - 3, 3);
+  }
+  if ((charset.size() > 5) &&
+      (charset[charset.size() - 5] == '-') &&
+      (charset[charset.size() - 4] == '1') &&
+      (charset[charset.size() - 3] == '9')) {
+    charset.erase(charset.size() - 5, 5);
+  }
+  if ((charset.size() > 5) &&
+      (charset[charset.size() - 5] == '-') &&
+      (charset[charset.size() - 4] == '2') &&
+      (charset[charset.size() - 3] == '0') &&
+      (charset[charset.size() - 2] == '0')) {
+    charset.erase(charset.size() - 5, 5);
+  }
+
+  // Truncate
+  if (charset.size() > 16) {
+    charset.resize(16);
+  }
+
+  return charset;
+}
+
+int GetHttpHeaderLength(const char* document_text, uint32 document_length) {
+  // HTTP headers end with cr lf cr lf
+  const char* end = FindSubstring(document_text,
+                                  document_text + document_length,
+                                  "\r\n\r\n");
+  return end
+    ? end - document_text + 4 // skip over the cr lf cr lf
+    : 0;
+}
+/*
+// Get top level domain from URL
+// Normalize: truncate to 16 chars and make lowercase
+string GetTLD(const char* url_str) {
+  // some of urls are escaped, we need to unescape them. Otherwise
+  // you will see the messy TLDs.
+  if (url_str == NULL) {
+    return string("");
+  }
+
+  string unescaped_url;
+  URL::UnescapeURL(url_str, strlen(url_str), &unescaped_url);
+  URL url(unescaped_url);
+  const char *hostname = url.host();
+  const char *lastdot = strrchr(hostname, '.');
+  if (lastdot == NULL) {
+    // no dot in host; maybe it's not a fully qualified host name
+    return "";
+  }
+
+  const char *tld_str = lastdot + 1;
+  // TLD can only have letters
+  for (const char *p = tld_str; *p != '\0'; ++p) {
+    if (!isalpha(*p)) {
+      return "";
+    }
+  }
+
+  string tld(tld_str);
+  // Truncate
+  if (tld.size() > 16) {
+    tld.resize(16);
+  }
+  StringToLowercase(&tld);
+
+  return tld;
+}
+
+// Get charset from HTTP headers
+// Normalize: truncate to 16 chars and make lowercase
+string GetCharsetFromHttp(const char* http, int http_len) {
+  if (FindSubstring(http, http + http_len, "charset")) {
+    string headers(http, http_len);
+    return GetCharset(headers);
+  }
+  return "";
+}
+
+// Get charset= from <meta> tag
+// Or get encoding= from <?xml?>  tag
+//   <?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
+// Normalize: truncate to 16 chars and make lowercase
+string GetCharsetFromMeta(const char* body, int body_len) {
+  const char* start = body;
+  const char* const end = start + body_len;
+  while (start < end) {
+    const char* meta = FindSubstring(start, end, "<meta ");
+    if (meta == NULL) {
+        break;
+    }
+    const char* endtag = FindSubstring(meta, end, ">");
+    if (endtag == NULL) {
+      break;
+    }
+    if (endtag - meta > 1024) {
+      endtag = meta + 1024;
+    }
+    const char* meta_end = endtag + 1;
+    if (FindSubstring(meta, meta_end, "charset") != NULL) {
+      return GetCharset(string(meta, meta_end - meta));
+    }
+    start = meta_end;
+  }
+
+  start = body;
+  while (start < end) {
+    const char* meta = FindSubstring(start, end, "<?xml ");
+    if (meta == NULL) {
+      break;
+    }
+    const char* endtag = FindSubstring(meta, end, ">");
+    if (endtag == NULL) {
+      break;
+    }
+    if (endtag - meta > 1024) {
+      endtag = meta + 1024;
+    }
+    const char* meta_end = endtag + 1;
+    if (FindSubstring(meta, meta_end, "encoding") != NULL) {
+      return GetCharset(string(meta, meta_end - meta));
+    }
+    start = meta_end;
+  }
+
+  return "";
+}
+*/