Experimental parser: merge to r19637

src/scanner.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 17 matching lines @@
 // Features shared by parsing and pre-parsing scanners.
 
 #include <cmath>
 
 #include "scanner.h"
 
 #include "../include/v8stdint.h"
 #include "char-predicates-inl.h"
 #include "conversions-inl.h"
 #include "list-inl.h"
+#include "v8.h"
 
 namespace v8 {
 namespace internal {
 
+
+#ifndef V8_USE_GENERATED_LEXER
 // ----------------------------------------------------------------------------
 // Scanner
 
 Scanner::Scanner(UnicodeCache* unicode_cache)
     : unicode_cache_(unicode_cache),
       octal_pos_(Location::invalid()),
       harmony_scoping_(false),
       harmony_modules_(false),
       harmony_numeric_literals_(false) { }
 
@@ skipping 187 matching lines @@
       next_.location.end_pos = pos + 1;
       Advance();
       return current_.token;
     }
   }
   Scan();
   return current_.token;
 }
 
 
-static inline bool IsByteOrderMark(uc32 c) {
+// TODO(yangguo): check whether this is actually necessary.
+static inline bool IsLittleEndianByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
   // Unicode character; this implies that in a Unicode context the
   // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF
   // character expressed in little-endian byte order (since it could
   // not be a U+FFFE character expressed in big-endian byte
   // order). Nevertheless, we check for it to be compatible with
   // Spidermonkey.
-  return c == 0xFEFF || c == 0xFFFE;
+  return c == 0xFFFE;
 }
 
 
 bool Scanner::SkipWhiteSpace() {
   int start_position = source_pos();
 
   while (true) {
-    // We treat byte-order marks (BOMs) as whitespace for better
-    // compatibility with Spidermonkey and other JavaScript engines.
-    while (unicode_cache_->IsWhiteSpace(c0_) || IsByteOrderMark(c0_)) {
-      // IsWhiteSpace() includes line terminators!
+    while (true) {
+      // Advance as long as character is a WhiteSpace or LineTerminator.
+      // Remember if the latter is the case.
       if (unicode_cache_->IsLineTerminator(c0_)) {
-        // Ignore line terminators, but remember them. This is necessary
-        // for automatic semicolon insertion.
         has_line_terminator_before_next_ = true;
+      } else if (!unicode_cache_->IsWhiteSpace(c0_) &&
+                 !IsLittleEndianByteOrderMark(c0_)) {
+        break;
       }
       Advance();
     }
 
     // If there is an HTML comment end '-->' at the beginning of a
     // line (with only whitespace in front of it), we treat the rest
     // of the line as a comment. This is in line with the way
     // SpiderMonkey handles it.
     if (c0_ == '-' && has_line_terminator_before_next_) {
       Advance();
@@ skipping 823 matching lines @@
       Advance();
     }
   }
   literal.Complete();
 
   next_.location.end_pos = source_pos() - 1;
   return true;
 }
 
 
+#endif
+
+
 int DuplicateFinder::AddAsciiSymbol(Vector<const char> key, int value) {
   return AddSymbol(Vector<const byte>::cast(key), true, value);
 }
 
 
 int DuplicateFinder::AddUtf16Symbol(Vector<const uint16_t> key, int value) {
   return AddSymbol(Vector<const byte>::cast(key), false, value);
 }
 
 
@@ skipping 116 matching lines @@
     }
     backing_store_.Add(static_cast<byte>((ascii_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<byte>(ascii_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 
 } }  // namespace v8::internal