Optimize phishing page term feature extraction.

base/string_piece.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 // Copied from strings/stringpiece.h with modifications
 //
 // A string-like object that points to a sized piece of memory.
 //
 // Functions or methods may use const StringPiece& parameters to accept either
 // a "const char*" or a "string" value that will be implicitly converted to
 // a StringPiece.  The implicit conversion means that it is often appropriate
 // to include this .h file in other files rather than forward-declaring
 // StringPiece as would be appropriate for most other Google classes.
 //
 // Systematic usage of StringPiece is encouraged as it will reduce unnecessary
 // conversions from "const char*" to "string" and back again.
 //
+// StringPiece16 is similar to StringPiece but for base::string16 instead of
+// std::string. We do not define as large of a subset of the STL functions
+// from basic_string as in StringPiece, but this can be changed if these
+// functions (find, find_first_of, etc.) are found to be useful in this context.
+//
 
 #ifndef BASE_STRING_PIECE_H_
 #define BASE_STRING_PIECE_H_
 #pragma once
 
 #include <string>
 
 #include "base/base_api.h"
 #include "base/basictypes.h"
+#include "base/hash_tables.h"
+#include "base/string16.h"
 
 namespace base {
 
 class BASE_API StringPiece {
  public:
   // standard STL container boilerplate
   typedef size_t size_type;
   typedef char value_type;
   typedef const char* pointer;
   typedef const char& reference;
@@ skipping 121 matching lines @@
 
   static int wordmemcmp(const char* p, const char* p2, size_type N) {
     return memcmp(p, p2, N);
   }
 
  private:
   const char*   ptr_;
   size_type     length_;
 };
 
+class BASE_API StringPiece16 {
+ public:
+  // standard STL container boilerplate
+  typedef size_t size_type;
+  typedef char16 value_type;
+  typedef const char16* pointer;
+  typedef const char16& reference;
+  typedef const char16& const_reference;
+  typedef ptrdiff_t difference_type;
+  typedef const char16* const_iterator;
+  typedef const char16* iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+
+ public:
+  // We provide non-explicit singleton constructors so users can pass
+  // in a "const char16*" or a "string16" wherever a "StringPiece16" is
+  // expected.
+  StringPiece16() : ptr_(NULL), length_(0) { }
+  StringPiece16(const char16* str)
+      : ptr_(str),
+        length_((str == NULL) ? 0 : string16::traits_type::length(str)) { }
+  StringPiece16(const string16& str)
+      : ptr_(str.data()), length_(str.size()) { }
+  StringPiece16(const char16* offset, size_type len)
+      : ptr_(offset), length_(len) { }
+
+  // data() may return a pointer to a buffer with embedded NULs, and the
+  // returned buffer may or may not be null terminated.  Therefore it is
+  // typically a mistake to pass data() to a routine that expects a NUL
+  // terminated string.
+  const char16* data() const { return ptr_; }
+  size_type size() const { return length_; }
+  size_type length() const { return length_; }
+  bool empty() const { return length_ == 0; }
+
+  void clear() {
+    ptr_ = NULL;
+    length_ = 0;
+  }
+  void set(const char16* data, size_type len) {
+    ptr_ = data;
+    length_ = len;
+  }
+  void set(const char16* str) {
+    ptr_ = str;
+    length_ = str ? string16::traits_type::length(str) : 0;
+  }
+  void set(const void* data, size_type len) {
+    ptr_ = reinterpret_cast<const char16*>(data);
+    length_ = len;
+  }
+
+  char16 operator[](size_type i) const { return ptr_[i]; }
+
+  string16 as_string16() const {
+    // StringPiece claims that this is bad when data() is NULL, but unittesting
+    // seems to say otherwise.
+    return string16(data(), size());
+  }
+
+  iterator begin() const { return ptr_; }
+  iterator end() const { return ptr_ + length_; }
+  const_reverse_iterator rbegin() const {
+    return const_reverse_iterator(ptr_ + length_);
+  }
+  const_reverse_iterator rend() const {
+    return const_reverse_iterator(ptr_);
+  }
+
+  size_type max_size() const { return length_; }
+  size_type capacity() const { return length_; }
+
+  static int wordmemcmp(const char16* p, const char16* p2, size_type N) {
+    return string16::traits_type::compare(p, p2, N);
+  }
+
+ private:
+  const char16*   ptr_;
+  size_type       length_;
+};
+
+
 BASE_API bool operator==(const StringPiece& x, const StringPiece& y);
 
 inline bool operator!=(const StringPiece& x, const StringPiece& y) {
   return !(x == y);
 }
 
 inline bool operator<(const StringPiece& x, const StringPiece& y) {
   const int r = StringPiece::wordmemcmp(
       x.data(), y.data(), (x.size() < y.size() ? x.size() : y.size()));
   return ((r < 0) || ((r == 0) && (x.size() < y.size())));
 }
 
 inline bool operator>(const StringPiece& x, const StringPiece& y) {
   return y < x;
 }
 
 inline bool operator<=(const StringPiece& x, const StringPiece& y) {
   return !(x > y);
 }
 
 inline bool operator>=(const StringPiece& x, const StringPiece& y) {
   return !(x < y);
 }
 
+inline bool operator==(const StringPiece16& x, const StringPiece16& y) {
+  if (x.size() != y.size())
+    return false;
+
+  return StringPiece16::wordmemcmp(x.data(), y.data(), x.size()) == 0;
+}
+
+inline bool operator!=(const StringPiece16& x, const StringPiece16& y) {
+  return !(x == y);
+}
+
+inline bool operator<(const StringPiece16& x, const StringPiece16& y) {
+  const int r = StringPiece16::wordmemcmp(
+      x.data(), y.data(), (x.size() < y.size() ? x.size() : y.size()));
+  return ((r < 0) || ((r == 0) && (x.size() < y.size())));
+}
+
+inline bool operator>(const StringPiece16& x, const StringPiece16& y) {
+  return y < x;
+}
+
+inline bool operator<=(const StringPiece16& x, const StringPiece16& y) {
+  return !(x > y);
+}
+
+inline bool operator>=(const StringPiece16& x, const StringPiece16& y) {
+  return !(x < y);
+}
+
 }  // namespace base
 
+// We provide appropriate hash functions so StringPiece16 can be used as a
+// key in hash sets and maps.
+
+// This hash function is copied from base/hash_tables.h. We don't use the
+// already defined string16 directly because it would require the string16
+// constructor to be called, which we don't want.
+#define HASH_WIDE_STRING_PIECE(wsp) \

[brettw, 2011/08/05 23:00:23]

It doesn't look like regular StringPiece has a hash function, and with a trivial
addition of the "type" here you can use this same macro definition:
#define HASH_STRING_PIECE(StringPieceType, wsp)
 ...for(StringPieceType::const_iterator...
to define hashes below for StringPiece. Can you do that?

[Garrett Casto, 2011/08/08 18:48:05]

Done.

+  std::size_t result = 0;                                               \
+  for (base::StringPiece16::const_iterator i = wsp.begin();           \

[brettw, 2011/08/05 23:00:23]

Fix whitespace.

[Garrett Casto, 2011/08/08 18:48:05]

Done.

+       i != wsp.end(); ++i)                                             \
+    result = (result * 131) + *i;                                       \
+  return result;                                                        \
+
+namespace BASE_HASH_NAMESPACE {
+#if defined(COMPILER_GCC)
+
+template<>
+struct hash<base::StringPiece16> {
+  std::size_t operator()(const base::StringPiece16& wsp) const {
+    HASH_WIDE_STRING_PIECE(wsp);
+  }
+};
+
+#elif defined(COMPILER_MSVC)
+
+inline size_t hash_value(const base::StringPiece16& wsp) {
+  HASH_WIDE_STRING_PIECE(wsp);
+}
+
+#endif  // COMPILER
+
+}  // namespace BASE_HASH_NAMESPACE
+
 #endif  // BASE_STRING_PIECE_H_