Optimize phishing page term feature extraction.

base/string_piece.h

Index: base/string_piece.h
diff --git a/base/string_piece.h b/base/string_piece.h
index 60380b125b70af287458b74e2e045ad386c98769..4225ac801b2ad5dac34a3eff66bf297f7a539937 100644
--- a/base/string_piece.h
+++ b/base/string_piece.h
@@ -14,6 +14,11 @@
 // Systematic usage of StringPiece is encouraged as it will reduce unnecessary
 // conversions from "const char*" to "string" and back again.
 //
+// StringPiece16 is similar to StringPiece but for base::string16 instead of
+// std::string. We do not define as large of a subset of the STL functions
+// from basic_string as in StringPiece, but this can be changed if these
+// functions (find, find_first_of, etc.) are found to be useful in this context.
+//
 
 #ifndef BASE_STRING_PIECE_H_
 #define BASE_STRING_PIECE_H_
@@ -23,6 +28,8 @@
 
 #include "base/base_api.h"
 #include "base/basictypes.h"
+#include "base/hash_tables.h"
+#include "base/string16.h"
 
 namespace base {
 
@@ -164,6 +171,89 @@ class BASE_API StringPiece {
   size_type     length_;
 };
 
+class BASE_API StringPiece16 {
+ public:
+  // standard STL container boilerplate
+  typedef size_t size_type;
+  typedef char16 value_type;
+  typedef const char16* pointer;
+  typedef const char16& reference;
+  typedef const char16& const_reference;
+  typedef ptrdiff_t difference_type;
+  typedef const char16* const_iterator;
+  typedef const char16* iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+
+ public:
+  // We provide non-explicit singleton constructors so users can pass
+  // in a "const char16*" or a "string16" wherever a "StringPiece16" is
+  // expected.
+  StringPiece16() : ptr_(NULL), length_(0) { }
+  StringPiece16(const char16* str)
+      : ptr_(str),
+        length_((str == NULL) ? 0 : string16::traits_type::length(str)) { }
+  StringPiece16(const string16& str)
+      : ptr_(str.data()), length_(str.size()) { }
+  StringPiece16(const char16* offset, size_type len)
+      : ptr_(offset), length_(len) { }
+
+  // data() may return a pointer to a buffer with embedded NULs, and the
+  // returned buffer may or may not be null terminated.  Therefore it is
+  // typically a mistake to pass data() to a routine that expects a NUL
+  // terminated string.
+  const char16* data() const { return ptr_; }
+  size_type size() const { return length_; }
+  size_type length() const { return length_; }
+  bool empty() const { return length_ == 0; }
+
+  void clear() {
+    ptr_ = NULL;
+    length_ = 0;
+  }
+  void set(const char16* data, size_type len) {
+    ptr_ = data;
+    length_ = len;
+  }
+  void set(const char16* str) {
+    ptr_ = str;
+    length_ = str ? string16::traits_type::length(str) : 0;
+  }
+  void set(const void* data, size_type len) {
+    ptr_ = reinterpret_cast<const char16*>(data);
+    length_ = len;
+  }
+
+  char16 operator[](size_type i) const { return ptr_[i]; }
+
+  string16 as_string16() const {
+    // StringPiece claims that this is bad when data() is NULL, but unittesting
+    // seems to say otherwise.
+    return string16(data(), size());
+  }
+
+  iterator begin() const { return ptr_; }
+  iterator end() const { return ptr_ + length_; }
+  const_reverse_iterator rbegin() const {
+    return const_reverse_iterator(ptr_ + length_);
+  }
+  const_reverse_iterator rend() const {
+    return const_reverse_iterator(ptr_);
+  }
+
+  size_type max_size() const { return length_; }
+  size_type capacity() const { return length_; }
+
+  static int wordmemcmp(const char16* p, const char16* p2, size_type N) {
+    return string16::traits_type::compare(p, p2, N);
+  }
+
+ private:
+  const char16*   ptr_;
+  size_type       length_;
+};
+
+
 BASE_API bool operator==(const StringPiece& x, const StringPiece& y);
 
 inline bool operator!=(const StringPiece& x, const StringPiece& y) {
@@ -188,6 +278,68 @@ inline bool operator>=(const StringPiece& x, const StringPiece& y) {
   return !(x < y);
 }
 
+inline bool operator==(const StringPiece16& x, const StringPiece16& y) {
+  if (x.size() != y.size())
+    return false;
+
+  return StringPiece16::wordmemcmp(x.data(), y.data(), x.size()) == 0;
+}
+
+inline bool operator!=(const StringPiece16& x, const StringPiece16& y) {
+  return !(x == y);
+}
+
+inline bool operator<(const StringPiece16& x, const StringPiece16& y) {
+  const int r = StringPiece16::wordmemcmp(
+      x.data(), y.data(), (x.size() < y.size() ? x.size() : y.size()));
+  return ((r < 0) || ((r == 0) && (x.size() < y.size())));
+}
+
+inline bool operator>(const StringPiece16& x, const StringPiece16& y) {
+  return y < x;
+}
+
+inline bool operator<=(const StringPiece16& x, const StringPiece16& y) {
+  return !(x > y);
+}
+
+inline bool operator>=(const StringPiece16& x, const StringPiece16& y) {
+  return !(x < y);
+}
+
 }  // namespace base
 
+// We provide appropriate hash functions so StringPiece16 can be used as a
+// key in hash sets and maps.
+
+// This hash function is copied from base/hash_tables.h. We don't use the
+// already defined string16 directly because it would require the string16
+// constructor to be called, which we don't want.
+#define HASH_WIDE_STRING_PIECE(wsp) \

[brettw, 2011/08/05 23:00:23]

It doesn't look like regular StringPiece has a hash function, and with a trivial
addition of the "type" here you can use this same macro definition:
#define HASH_STRING_PIECE(StringPieceType, wsp)
 ...for(StringPieceType::const_iterator...
to define hashes below for StringPiece. Can you do that?

[Garrett Casto, 2011/08/08 18:48:05]

Done.

+  std::size_t result = 0;                                               \
+  for (base::StringPiece16::const_iterator i = wsp.begin();           \

[brettw, 2011/08/05 23:00:23]

Fix whitespace.

[Garrett Casto, 2011/08/08 18:48:05]

Done.

+       i != wsp.end(); ++i)                                             \
+    result = (result * 131) + *i;                                       \
+  return result;                                                        \
+
+namespace BASE_HASH_NAMESPACE {
+#if defined(COMPILER_GCC)
+
+template<>
+struct hash<base::StringPiece16> {
+  std::size_t operator()(const base::StringPiece16& wsp) const {
+    HASH_WIDE_STRING_PIECE(wsp);
+  }
+};
+
+#elif defined(COMPILER_MSVC)
+
+inline size_t hash_value(const base::StringPiece16& wsp) {
+  HASH_WIDE_STRING_PIECE(wsp);
+}
+
+#endif  // COMPILER
+
+}  // namespace BASE_HASH_NAMESPACE
+
 #endif  // BASE_STRING_PIECE_H_