Push version 2.5.7 to trunk....

src/scanner-base.h

Index: src/scanner-base.h
===================================================================
--- src/scanner-base.h	(revision 5846)
+++ src/scanner-base.h	(working copy)
@@ -30,12 +30,56 @@
 #ifndef V8_SCANNER_BASE_H_
 #define V8_SCANNER_BASE_H_
 
+#include "globals.h"
+#include "checks.h"
+#include "allocation.h"
 #include "token.h"
-#include "unicode.h"
+#include "unicode-inl.h"
+#include "char-predicates.h"
+#include "utils.h"
 
 namespace v8 {
 namespace internal {
 
+// Interface through which the scanner reads characters from the input source.
+class UTF16Buffer {
+ public:
+  UTF16Buffer();
+  virtual ~UTF16Buffer() {}
+
+  virtual void PushBack(uc32 ch) = 0;
+  // Returns a value < 0 when the buffer end is reached.
+  virtual uc32 Advance() = 0;
+  virtual void SeekForward(int pos) = 0;
+
+  int pos() const { return pos_; }
+
+ protected:
+  int pos_;  // Current position in the buffer.
+  int end_;  // Position where scanning should stop (EOF).
+};
+
+
+class ScannerConstants : AllStatic {
+ public:
+  typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
+
+  static StaticResource<Utf8Decoder>* utf8_decoder() {
+    return &utf8_decoder_;
+  }
+
+  static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart;
+  static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart;
+  static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator;
+  static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace;
+
+  static bool IsIdentifier(unibrow::CharacterStream* buffer);
+
+ private:
+  static StaticResource<Utf8Decoder> utf8_decoder_;
+};
+
+
 class KeywordMatcher {
 //  Incrementally recognize keywords.
 //
@@ -45,7 +89,8 @@
 //      return switch this throw true try typeof var void while with
 //
 //  *: Actually "future reserved keywords". These are the only ones we
-//     recognized, the remaining are allowed as identifiers.
+//     recognize, the remaining are allowed as identifiers.
+//     In ES5 strict mode, we should disallow all reserved keywords.
  public:
   KeywordMatcher()
       : state_(INITIAL),
@@ -156,10 +201,6 @@
 };
 
 
-
-
-
-
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_BASE_H_