Experimental parser: add UnicodeCache.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 13 matching lines @@
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+// FIXME: some of this is probably not needed.
+#include "allocation.h"
+#include "ast.h"
+#include "preparse-data-format.h"
+#include "preparse-data.h"
+#include "scopes.h"
+#include "preparser.h"
+#include "api.h"
+#include "ast.h"
+#include "bootstrapper.h"
+#include "char-predicates-inl.h"
+#include "codegen.h"
+#include "compiler.h"
+#include "func-name-inferrer.h"
+#include "messages.h"
+#include "parser.h"
+#include "platform.h"
+#include "preparser.h"
+#include "runtime.h"
+#include "scanner-character-streams.h"
+#include "scopeinfo.h"
+#include "string-stream.h"
+
+
 // TODO:
 // - SpiderMonkey compatibility hack: "  --> something" is treated
 //   as a single line comment.
 // - Run-time lexing modifications: harmony number literals, keywords depending
 //   on harmony_modules, harmony_scoping
 // - Escaping the string literals (like the baseline does)
 // - Error recovery after illegal tokens.
 
 enum Condition {
   kConditionNormal,
@@ skipping 25 matching lines @@
   #define O_BINARY 0
 #endif
 
 #endif //  defined(WIN32)
 
 #include "experimental-scanner.h"
 #include "lexer.h"
 
 using namespace v8::internal;
 
+namespace {
+
+inline int HexValue(uc32 c) {
+  c -= '0';
+  if (static_cast<unsigned>(c) <= 9) return c;
+  c = (c | 0x20) - ('a' - '0');  // detect 0x11..0x16 and 0x31..0x36.
+  if (static_cast<unsigned>(c) <= 5) return c + 10;
+  return -1;
+}
+
+}
+
 #define PUSH_TOKEN(T) { send(T); SKIP(); }
 #define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); }
 #define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;}
 #define PUSH_LINE_TERMINATOR() { SKIP(); }
 #define TERMINATE_ILLEGAL() { send(Token::ILLEGAL); send(Token::EOS); return 1; }
 
-PushScanner::PushScanner(ExperimentalScanner* sink)
-: eof_(false),
+#define YYCTYPE uint8_t
+
+PushScanner::PushScanner(ExperimentalScanner* sink, UnicodeCache* unicode_cache)
+: unicode_cache_(unicode_cache),
+  eof_(false),
   state_(-1),
   condition_(kConditionNormal),
   limit_(NULL),
   start_(NULL),
   cursor_(NULL),
   marker_(NULL),
   real_start_(0),
   buffer_(NULL),
   buffer_end_(NULL),
   yych(0),
   yyaccept(0),
   sink_(sink) {
 
 }
 
 PushScanner::~PushScanner() {
 }
 
+
+uc32 PushScanner::ScanHexNumber(int length) {
+  // We have seen \uXXXX, let's see what it is.
+  // FIXME: we never end up in here if only a subset of the 4 chars are valid
+  // hex digits -> handle the case where they're not.
+  uc32 x = 0;
+  for (YYCTYPE* s = cursor_ - length; s != cursor_; ++s) {
+    int d = HexValue(*s);
+    if (d < 0) {
+      return -1;
+    }
+    x = x * 16 + d;
+  }
+  return x;
+}
+
+
+bool PushScanner::ValidIdentifierPart() {
+  return unicode_cache_->IsIdentifierPart(ScanHexNumber(4));
+}
+
+bool PushScanner::ValidIdentifierStart() {
+  return unicode_cache_->IsIdentifierStart(ScanHexNumber(4));
+}
+
 void PushScanner::send(Token::Value token) {
   int beg = (start_ - buffer_) + real_start_;
   int end = (cursor_ - buffer_) + real_start_;
   if (FLAG_trace_lexer) {
     printf("got %s at (%d, %d): ", Token::Name(token), beg, end);
     for (uint8_t* s = start_; s != cursor_; s++) printf("%c", (char)*s);
     printf(".\n");
   }
   sink_->Record(token, beg, end);
 }
@@ skipping 66 matching lines @@
   if (FLAG_trace_lexer) {
     printf("Starting a round; state_: %d, condition_: %d\n", state_, condition_);
   }
 
   /*!re2c
     re2c:indent:top = 1;
     re2c:yych:conversion = 0;
     re2c:condenumprefix = kCondition;
     re2c:define:YYCONDTYPE = Condition;
     re2c:define:YYCURSOR = cursor_;
-    re2c:define:YYCTYPE = uint8_t;
     re2c:define:YYLIMIT = limit_;
     re2c:define:YYMARKER = marker_;
 
     eof = "\000";
     any = [\000-\377];
     whitespace_char = [ \t\v\f\r];
     whitespace = whitespace_char+;
     identifier_start_ = [$_a-zA-Z];
     identifier_char = [$_a-zA-Z0-9];
     not_identifier_char = any\identifier_char\[\\];
@@ skipping 110 matching lines @@
     <Normal> "~"           { PUSH_TOKEN(Token::BIT_NOT); }
     <Normal> ","           { PUSH_TOKEN(Token::COMMA); }
 
     <Normal> line_terminator  { PUSH_LINE_TERMINATOR(); }
     <Normal> whitespace       { SKIP(); }
 
     <Normal> ["]           :=> DoubleQuoteString
     <Normal> [']           :=> SingleQuoteString
 
     <Normal> identifier_start_    :=> Identifier
-    <Normal> "\\u0000"            :=> IdentifierIllegal
-    <Normal> "\\u" [0-9a-fA-F]{4}    :=> Identifier
+    <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); goto yy0; } YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); start_ = cursor_;  goto yy0; }
     <Normal> "\\"                 { PUSH_TOKEN(Token::ILLEGAL); }
 
     <Normal> eof           { PUSH_EOF_AND_RETURN();}
     <Normal> any           { PUSH_TOKEN(Token::ILLEGAL); }
 
     <DoubleQuoteString> "\\\\"  { goto yy0; }
     <DoubleQuoteString> "\\\""  { goto yy0; }
     <DoubleQuoteString> '"'     { PUSH_TOKEN(Token::STRING);}
     <DoubleQuoteString> "\\" "\n" "\r"? { goto yy0; }
     <DoubleQuoteString> "\\" "\r" "\n"? { goto yy0; }
     <DoubleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <DoubleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <DoubleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <DoubleQuoteString> any     { goto yy0; }
 
     <SingleQuoteString> "\\\\"  { goto yy0; }
     <SingleQuoteString> "\\'"   { goto yy0; }
     <SingleQuoteString> "'"     { PUSH_TOKEN(Token::STRING);}
     <SingleQuoteString> "\\" "\n" "\r"? { goto yy0; }
     <SingleQuoteString> "\\" "\r" "\n"? { goto yy0; }
     <SingleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <SingleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <SingleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <SingleQuoteString> any     { goto yy0; }
 
     <Identifier> identifier_char+  { goto yy0; }
-    <Identifier> "\\u0000"         :=> IdentifierIllegal
-    <Identifier> "\\u" [0-9a-fA-F]{4} { goto yy0; }
+    <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) goto yy0; YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); }
     <Identifier> "\\"              { PUSH_TOKEN(Token::ILLEGAL); }
     <Identifier> any               { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
 
     <IdentifierIllegal> identifier_char+  { goto yy0; }
-    <IdentifierIllegal> "\\"+              { goto yy0; }
+    <IdentifierIllegal> "\\"+             { goto yy0; }
     <IdentifierIllegal> any               { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
 
     <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> eof             { PUSH_TOKEN(Token::EOS); }
     <SingleLineComment> any             { goto yy0; }
 
     <MultiLineComment> [*][//]  { PUSH_LINE_TERMINATOR();}
     <MultiLineComment> eof      { TERMINATE_ILLEGAL(); }
     <MultiLineComment> any      { goto yy0; }
 
@@ skipping 28 matching lines @@
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }