Push version 2.5.7 to trunk....

src/scanner-base.h

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 12 matching lines @@
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #ifndef V8_SCANNER_BASE_H_
 #define V8_SCANNER_BASE_H_
 
+#include "globals.h"
+#include "checks.h"
+#include "allocation.h"
 #include "token.h"
-#include "unicode.h"
+#include "unicode-inl.h"
+#include "char-predicates.h"
+#include "utils.h"
 
 namespace v8 {
 namespace internal {
 
+// Interface through which the scanner reads characters from the input source.
+class UTF16Buffer {
+ public:
+  UTF16Buffer();
+  virtual ~UTF16Buffer() {}
+
+  virtual void PushBack(uc32 ch) = 0;
+  // Returns a value < 0 when the buffer end is reached.
+  virtual uc32 Advance() = 0;
+  virtual void SeekForward(int pos) = 0;
+
+  int pos() const { return pos_; }
+
+ protected:
+  int pos_;  // Current position in the buffer.
+  int end_;  // Position where scanning should stop (EOF).
+};
+
+
+class ScannerConstants : AllStatic {
+ public:
+  typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
+
+  static StaticResource<Utf8Decoder>* utf8_decoder() {
+    return &utf8_decoder_;
+  }
+
+  static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart;
+  static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart;
+  static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator;
+  static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace;
+
+  static bool IsIdentifier(unibrow::CharacterStream* buffer);
+
+ private:
+  static StaticResource<Utf8Decoder> utf8_decoder_;
+};
+
+
 class KeywordMatcher {
 //  Incrementally recognize keywords.
 //
 //  Recognized keywords:
 //      break case catch const* continue debugger* default delete do else
 //      finally false for function if in instanceof native* new null
 //      return switch this throw true try typeof var void while with
 //
 //  *: Actually "future reserved keywords". These are the only ones we
-//     recognized, the remaining are allowed as identifiers.
+//     recognize, the remaining are allowed as identifiers.
+//     In ES5 strict mode, we should disallow all reserved keywords.
  public:
   KeywordMatcher()
       : state_(INITIAL),
         token_(Token::IDENTIFIER),
         keyword_(NULL),
         counter_(0),
         keyword_token_(Token::ILLEGAL) {}
 
   Token::Value token() { return token_; }
 
@@ skipping 90 matching lines @@
   Token::Value token_;
 
   // Matching a specific keyword string (there is only one possible valid
   // keyword with the current prefix).
   const char* keyword_;
   int counter_;
   Token::Value keyword_token_;
 };
 
 
-
-
-
-
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_BASE_H_