Simpler (and a bit faster) keyword matcher

test/cctest/test-parsing.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 24 matching lines @@
 #include "token.h"
 #include "scanner.h"
 #include "parser.h"
 #include "utils.h"
 #include "execution.h"
 #include "preparser.h"
 #include "cctest.h"
 
 namespace i = ::v8::internal;
 
-TEST(KeywordMatcher) {
+TEST(ScanKeywords) {
   struct KeywordToken {
     const char* keyword;
     i::Token::Value token;
   };
 
   static const KeywordToken keywords[] = {
 #define KEYWORD(t, s, d) { s, i::Token::t },
-#define IGNORE(t, s, d)  /* */
-      TOKEN_LIST(IGNORE, KEYWORD, IGNORE)
+      TOKEN_LIST(IGNORE_TOKEN, KEYWORD)
 #undef KEYWORD
       { NULL, i::Token::IDENTIFIER }
   };
 
-  static const char* future_keywords[] = {
-#define FUTURE(t, s, d) s,
-      TOKEN_LIST(IGNORE, IGNORE, FUTURE)
-#undef FUTURE
-#undef IGNORE
-      NULL
-  };
-
   KeywordToken key_token;
+  i::UnicodeCache unicode_cache;
+  i::byte buffer[32];
   for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) {
-    i::KeywordMatcher matcher;
-    const char* keyword = key_token.keyword;
-    int length = i::StrLength(keyword);
-    for (int j = 0; j < length; j++) {
-      if (key_token.token == i::Token::INSTANCEOF && j == 2) {
-        // "in" is a prefix of "instanceof". It's the only keyword
-        // that is a prefix of another.
-        CHECK_EQ(i::Token::IN, matcher.token());
-      } else {
-        CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
-      }
-      matcher.AddChar(keyword[j]);
+    const i::byte* keyword =
+        reinterpret_cast<const i::byte*>(key_token.keyword);
+    int length = i::StrLength(key_token.keyword);
+    CHECK(static_cast<int>(sizeof(buffer)) >= length);
+    {
+      i::Utf8ToUC16CharacterStream stream(keyword, length);
+      i::JavaScriptScanner scanner(&unicode_cache);
+      scanner.Initialize(&stream);
+      CHECK_EQ(key_token.token, scanner.Next());
+      CHECK_EQ(i::Token::EOS, scanner.Next());
     }
-    CHECK_EQ(key_token.token, matcher.token());
-    // Adding more characters will make keyword matching fail.
-    matcher.AddChar('z');
-    CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
-    // Adding a keyword later will not make it match again.
-    matcher.AddChar('i');
-    matcher.AddChar('f');
-    CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
+    // Removing characters will make keyword matching fail.
+    {
+      i::Utf8ToUC16CharacterStream stream(keyword, length - 1);
+      i::JavaScriptScanner scanner(&unicode_cache);
+      scanner.Initialize(&stream);
+      CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
+      CHECK_EQ(i::Token::EOS, scanner.Next());
+    }
+    // Adding characters will make keyword matching fail.
+    {
+      memmove(buffer, keyword, length);

[Lasse Reichstein, 2011/08/09 12:48:02]

Could you try appending all three of a letter, a digit and an underscore?

[Vitaly Repeshko, 2011/08/09 13:08:01]

Done.

+      buffer[length] = '_';
+      i::Utf8ToUC16CharacterStream stream(buffer, length + 1);
+      i::JavaScriptScanner scanner(&unicode_cache);
+      scanner.Initialize(&stream);
+      CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
+      CHECK_EQ(i::Token::EOS, scanner.Next());
+    }
+    // Replacing characters will make keyword matching fail.
+    {
+      memmove(buffer, keyword, length);
+      buffer[length - 1] = '_';
+      i::Utf8ToUC16CharacterStream stream(buffer, length);
+      i::JavaScriptScanner scanner(&unicode_cache);
+      scanner.Initialize(&stream);
+      CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
+      CHECK_EQ(i::Token::EOS, scanner.Next());
+    }
   }
-
-  // Future keywords are not recognized.
-  const char* future_keyword;
-  for (int i = 0; (future_keyword = future_keywords[i]) != NULL; i++) {
-    i::KeywordMatcher matcher;
-    int length = i::StrLength(future_keyword);
-    for (int j = 0; j < length; j++) {
-      matcher.AddChar(future_keyword[j]);
-    }
-    CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
-  }
-
-  // Zero isn't ignored at first.
-  i::KeywordMatcher bad_start;
-  bad_start.AddChar(0);
-  CHECK_EQ(i::Token::IDENTIFIER, bad_start.token());
-  bad_start.AddChar('i');
-  bad_start.AddChar('f');
-  CHECK_EQ(i::Token::IDENTIFIER, bad_start.token());
-
-  // Zero isn't ignored at end.
-  i::KeywordMatcher bad_end;
-  bad_end.AddChar('i');
-  bad_end.AddChar('f');
-  CHECK_EQ(i::Token::IF, bad_end.token());
-  bad_end.AddChar(0);
-  CHECK_EQ(i::Token::IDENTIFIER, bad_end.token());
-
-  // Case isn't ignored.
-  i::KeywordMatcher bad_case;
-  bad_case.AddChar('i');
-  bad_case.AddChar('F');
-  CHECK_EQ(i::Token::IDENTIFIER, bad_case.token());
-
-  // If we mark it as failure, continuing won't help.
-  i::KeywordMatcher full_stop;
-  full_stop.AddChar('i');
-  CHECK_EQ(i::Token::IDENTIFIER, full_stop.token());
-  full_stop.Fail();
-  CHECK_EQ(i::Token::IDENTIFIER, full_stop.token());
-  full_stop.AddChar('f');
-  CHECK_EQ(i::Token::IDENTIFIER, full_stop.token());
 }
 
 
 TEST(ScanHTMLEndComments) {
   v8::V8::Initialize();
 
   // Regression test. See:
   //    http://code.google.com/p/chromium/issues/detail?id=53548
   // Tests that --> is correctly interpreted as comment-to-end-of-line if there
   // is only whitespace before it on the line (with comments considered as
@@ skipping 586 matching lines @@
   TestScanRegExp("/[\\u12]/flipperwald", "[\\u12]");
   TestScanRegExp("/[\\u123]/flipperwald", "[\\u123]");
   // Escaped ']'s wont end the character class.
   TestScanRegExp("/[\\]/]/flipperwald", "[\\]/]");
   // Escaped slashes are not terminating.
   TestScanRegExp("/\\//flipperwald", "\\/");
   // Starting with '=' works too.
   TestScanRegExp("/=/", "=");
   TestScanRegExp("/=?/", "=?");
 }