[wasm][asm.js] Adding custom asm.js lexer.

test/unittests/asmjs/asm-lexer-unittest.cc

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/asmjs/asm-lexer.h"
+#include "src/objects.h"
+#include "src/parsing/scanner-character-streams.h"
+#include "src/parsing/scanner.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace v8 {
+namespace internal {
+
+#define TOK(t) AsmJsLexer::kToken_##t
+
+class AsmJsLexerTest : public ::testing::Test {
+ protected:
+  void SetupSource(const char* source) {
+    lexer.SetStream(ScannerStream::ForTesting(source));
+  }
+
+  void Skip(AsmJsLexer::token_t t) {
+    CHECK_EQ(t, lexer.Token());
+    lexer.Next();
+  }
+
+  void SkipGlobal() {
+    CHECK(lexer.IsGlobal());
+    lexer.Next();
+  }
+
+  void SkipLocal() {
+    CHECK(lexer.IsLocal());
+    lexer.Next();
+  }
+
+  void CheckForEnd() { CHECK(lexer.Token() == AsmJsLexer::kEndOfInput); }
+
+  AsmJsLexer lexer;
+};
+
+TEST_F(AsmJsLexerTest, SimpleFunction) {
+  SetupSource("function foo() { return; }");
+  Skip(TOK(function));
+  DCHECK_EQ("foo", lexer.GetIdentifierString());
+  SkipGlobal();
+  Skip('(');
+  Skip(')');
+  Skip('{');
+  // clang-format off
+  Skip(TOK(return));
+  // clang-format on
+  Skip(';');
+  Skip('}');
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, JSKeywords) {
+  SetupSource(
+      "arguments break case const continue\n"
+      "default do else eval for function\n"
+      "if new return switch var while\n");
+  Skip(TOK(arguments));
+  Skip(TOK(break));
+  Skip(TOK(case));
+  Skip(TOK(const));
+  Skip(TOK(continue));
+  Skip(TOK(default));
+  Skip(TOK(do));
+  Skip(TOK(else));
+  Skip(TOK(eval));
+  Skip(TOK(for));
+  Skip(TOK(function));
+  Skip(TOK(if));
+  Skip(TOK(new));
+  // clang-format off
+  Skip(TOK(return));
+  // clang-format on
+  Skip(TOK(switch));
+  Skip(TOK(var));
+  Skip(TOK(while));
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, JSOperatorsSpread) {
+  SetupSource(
+      "+ - * / % & | ^ ~ << >> >>>\n"
+      "< > <= >= == !=\n");
+  Skip('+');
+  Skip('-');
+  Skip('*');
+  Skip('/');
+  Skip('%');
+  Skip('&');
+  Skip('|');
+  Skip('^');
+  Skip('~');
+  Skip(TOK(SHL));
+  Skip(TOK(SAR));
+  Skip(TOK(SHR));
+  Skip('<');
+  Skip('>');
+  Skip(TOK(LE));
+  Skip(TOK(GE));
+  Skip(TOK(EQ));
+  Skip(TOK(NE));
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, JSOperatorsTight) {
+  SetupSource(
+      "+-*/%&|^~<<>> >>>\n"
+      "<><=>= ==!=\n");
+  Skip('+');
+  Skip('-');
+  Skip('*');
+  Skip('/');
+  Skip('%');
+  Skip('&');
+  Skip('|');
+  Skip('^');
+  Skip('~');
+  Skip(TOK(SHL));
+  Skip(TOK(SAR));
+  Skip(TOK(SHR));
+  Skip('<');
+  Skip('>');
+  Skip(TOK(LE));
+  Skip(TOK(GE));
+  Skip(TOK(EQ));
+  Skip(TOK(NE));
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, UsesOfAsm) {
+  SetupSource("'use asm' \"use asm\"\n");
+  Skip(TOK(UseAsm));
+  Skip(TOK(UseAsm));
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, DefaultGlobalScope) {
+  SetupSource("var x = x + x;");
+  Skip(TOK(var));
+  CHECK_EQ("x", lexer.GetIdentifierString());
+  AsmJsLexer::token_t x = lexer.Token();
+  SkipGlobal();
+  Skip('=');
+  Skip(x);
+  Skip('+');
+  Skip(x);
+  Skip(';');
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, GlobalScope) {
+  SetupSource("var x = x + x;");
+  lexer.EnterGlobalScope();
+  Skip(TOK(var));
+  CHECK_EQ("x", lexer.GetIdentifierString());
+  AsmJsLexer::token_t x = lexer.Token();
+  SkipGlobal();
+  Skip('=');
+  Skip(x);
+  Skip('+');
+  Skip(x);
+  Skip(';');
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, LocalScope) {
+  SetupSource("var x = x + x;");
+  lexer.EnterLocalScope();
+  Skip(TOK(var));
+  CHECK_EQ("x", lexer.GetIdentifierString());
+  AsmJsLexer::token_t x = lexer.Token();
+  SkipLocal();
+  Skip('=');
+  Skip(x);
+  Skip('+');
+  Skip(x);
+  Skip(';');
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, Numbers) {
+  SetupSource("1 1.2 0x1f 1.e3");
+
+  CHECK(lexer.IsUnsigned());
+  CHECK_EQ(1, lexer.AsUnsigned());
+  lexer.Next();
+
+  CHECK(lexer.IsDouble());
+  CHECK_EQ(1.2, lexer.AsDouble());
+  lexer.Next();
+
+  CHECK(lexer.IsUnsigned());
+  CHECK_EQ(31, lexer.AsUnsigned());
+  lexer.Next();
+
+  CHECK(lexer.IsDouble());
+  CHECK_EQ(1.0e3, lexer.AsDouble());
+  lexer.Next();
+
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, Rewind1) {
+  SetupSource("+ - * /");
+  Skip('+');
+  lexer.Rewind();
+  Skip('+');
+  Skip('-');
+  lexer.Rewind();
+  Skip('-');
+  Skip('*');
+  lexer.Rewind();
+  Skip('*');
+  Skip('/');
+  lexer.Rewind();
+  Skip('/');
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, Comments) {
+  SetupSource(
+      "var // This is a test /* */\n"
+      "var /* test *** test */\n"
+      "function /* this */ ^");
+  Skip(TOK(var));
+  Skip(TOK(var));
+  Skip(TOK(function));
+  Skip('^');
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, Seeking) {
+  SetupSource("var eval do arguments function break\n");
+  Skip(TOK(var));
+  int old_pos = lexer.GetPosition();
+  Skip(TOK(eval));
+  Skip(TOK(do));
+  Skip(TOK(arguments));
+  lexer.Rewind();
+  Skip(TOK(arguments));
+  lexer.Rewind();
+  lexer.Seek(old_pos);
+  Skip(TOK(do));
+  Skip(TOK(arguments));
+  Skip(TOK(function));
+  Skip(TOK(break));
+  CheckForEnd();
+}
+
+TEST_F(AsmJsLexerTest, Newlines) {
+  SetupSource(
+      "var x = 1\n"
+      "var y = 2\n");
+  Skip(TOK(var));
+  lexer.Next();
+  Skip('=');
+  lexer.Next();
+  CHECK(lexer.IsPrecededByNewline());
+  Skip(TOK(var));
+  lexer.Next();
+  Skip('=');
+  lexer.Next();
+  CHECK(lexer.IsPrecededByNewline());
+  CheckForEnd();
+}
+
+}  // namespace internal
+}  // namespace v8