[modules] Disable HTML-like comments

test/cctest/test-parsing.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 53 matching lines @@
   i::UnicodeCache unicode_cache;
   i::byte buffer[32];
   for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) {
     const i::byte* keyword =
         reinterpret_cast<const i::byte*>(key_token.keyword);
     int length = i::StrLength(key_token.keyword);
     CHECK(static_cast<int>(sizeof(buffer)) >= length);
     {
       i::Utf8ToUtf16CharacterStream stream(keyword, length);
       i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(&stream);
+      scanner.Initialize(&stream, true);
       CHECK_EQ(key_token.token, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Removing characters will make keyword matching fail.
     {
       i::Utf8ToUtf16CharacterStream stream(keyword, length - 1);
       i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(&stream);
+      scanner.Initialize(&stream, true);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Adding characters will make keyword matching fail.
     static const char chars_to_append[] = { 'z', '0', '_' };
     for (int j = 0; j < static_cast<int>(arraysize(chars_to_append)); ++j) {
       i::MemMove(buffer, keyword, length);
       buffer[length] = chars_to_append[j];
       i::Utf8ToUtf16CharacterStream stream(buffer, length + 1);
       i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(&stream);
+      scanner.Initialize(&stream, true);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Replacing characters will make keyword matching fail.
     {
       i::MemMove(buffer, keyword, length);
       buffer[length - 1] = '_';
       i::Utf8ToUtf16CharacterStream stream(buffer, length);
       i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(&stream);
+      scanner.Initialize(&stream, true);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
   }
 }
 
 
-TEST(ScanHTMLEndComments) {
-  v8::V8::Initialize();
-  v8::Isolate* isolate = CcTest::isolate();
-  v8::HandleScope handles(isolate);
-
-  // Regression test. See:
-  //    http://code.google.com/p/chromium/issues/detail?id=53548
-  // Tests that --> is correctly interpreted as comment-to-end-of-line if there
-  // is only whitespace before it on the line (with comments considered as
-  // whitespace, even a multiline-comment containing a newline).
-  // This was not the case if it occurred before the first real token
-  // in the input.
-  const char* tests[] = {
-      // Before first real token.
-      "--> is eol-comment\nvar y = 37;\n",
-      "\n --> is eol-comment\nvar y = 37;\n",
-      "/* precomment */ --> is eol-comment\nvar y = 37;\n",
-      "\n/* precomment */ --> is eol-comment\nvar y = 37;\n",
-      // After first real token.
-      "var x = 42;\n--> is eol-comment\nvar y = 37;\n",
-      "var x = 42;\n/* precomment */ --> is eol-comment\nvar y = 37;\n",
-      NULL
-  };
-
-  const char* fail_tests[] = {
-      "x --> is eol-comment\nvar y = 37;\n",
-      "\"\\n\" --> is eol-comment\nvar y = 37;\n",
-      "x/* precomment */ --> is eol-comment\nvar y = 37;\n",
-      "x/* precomment\n */ --> is eol-comment\nvar y = 37;\n",
-      "var x = 42; --> is eol-comment\nvar y = 37;\n",
-      "var x = 42; /* precomment\n */ --> is eol-comment\nvar y = 37;\n",
-      NULL
-  };
-
-  // Parser/Scanner needs a stack limit.
-  CcTest::i_isolate()->stack_guard()->SetStackLimit(
-      i::GetCurrentStackPosition() - 128 * 1024);
-  uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
-  for (int i = 0; tests[i]; i++) {
-    const i::byte* source =
-        reinterpret_cast<const i::byte*>(tests[i]);
-    i::Utf8ToUtf16CharacterStream stream(source, i::StrLength(tests[i]));
-    i::CompleteParserRecorder log;
-    i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-    scanner.Initialize(&stream);
-    i::Zone zone(CcTest::i_isolate()->allocator());
-    i::AstValueFactory ast_value_factory(
-        &zone, CcTest::i_isolate()->heap()->HashSeed());
-    i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
-                           stack_limit);
-    preparser.set_allow_lazy(true);
-    i::PreParser::PreParseResult result = preparser.PreParseProgram();
-    CHECK_EQ(i::PreParser::kPreParseSuccess, result);
-    CHECK(!log.HasError());
-  }
-
-  for (int i = 0; fail_tests[i]; i++) {
-    const i::byte* source =
-        reinterpret_cast<const i::byte*>(fail_tests[i]);
-    i::Utf8ToUtf16CharacterStream stream(source, i::StrLength(fail_tests[i]));
-    i::CompleteParserRecorder log;
-    i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-    scanner.Initialize(&stream);
-    i::Zone zone(CcTest::i_isolate()->allocator());
-    i::AstValueFactory ast_value_factory(
-        &zone, CcTest::i_isolate()->heap()->HashSeed());
-    i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
-                           stack_limit);
-    preparser.set_allow_lazy(true);
-    i::PreParser::PreParseResult result = preparser.PreParseProgram();
-    // Even in the case of a syntax error, kPreParseSuccess is returned.
-    CHECK_EQ(i::PreParser::kPreParseSuccess, result);
-    CHECK(log.HasError());
-  }
-}
-
-
 class ScriptResource : public v8::String::ExternalOneByteStringResource {
  public:
   ScriptResource(const char* data, size_t length)
       : data_(data), length_(length) { }
 
   const char* data() const { return data_; }
   size_t length() const { return length_; }
 
  private:
   const char* data_;
@@ skipping 125 matching lines @@
   };
 
   uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     const char* program = programs[i];
     i::Utf8ToUtf16CharacterStream stream(
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
     i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-    scanner.Initialize(&stream);
+    scanner.Initialize(&stream, true);
 
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->heap()->HashSeed());
     i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                            stack_limit);
     preparser.set_allow_lazy(true);
     preparser.set_allow_natives(true);
     i::PreParser::PreParseResult result = preparser.PreParseProgram();
     CHECK_EQ(i::PreParser::kPreParseSuccess, result);
@@ skipping 15 matching lines @@
   };
 
   uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     const char* program = programs[i];
     i::Utf8ToUtf16CharacterStream stream(
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
     i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-    scanner.Initialize(&stream);
+    scanner.Initialize(&stream, true);
 
     // Preparser defaults to disallowing natives syntax.
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->heap()->HashSeed());
     i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                            stack_limit);
     preparser.set_allow_lazy(true);
     i::PreParser::PreParseResult result = preparser.PreParseProgram();
     CHECK_EQ(i::PreParser::kPreParseSuccess, result);
@@ skipping 50 matching lines @@
   // Fails parsing expecting an identifier after "function".
   // Before fix, didn't check *ok after Expect(Token::Identifier, ok),
   // and then used the invalid currently scanned literal. This always
   // failed in debug mode, and sometimes crashed in release mode.
 
   i::Utf8ToUtf16CharacterStream stream(
       reinterpret_cast<const i::byte*>(program),
       static_cast<unsigned>(strlen(program)));
   i::CompleteParserRecorder log;
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(&stream);
+  scanner.Initialize(&stream, true);
   i::Zone zone(CcTest::i_isolate()->allocator());
   i::AstValueFactory ast_value_factory(&zone,
                                        CcTest::i_isolate()->heap()->HashSeed());
   i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                          CcTest::i_isolate()->stack_guard()->real_climit());
   preparser.set_allow_lazy(true);
   i::PreParser::PreParseResult result = preparser.PreParseProgram();
   // Even in the case of a syntax error, kPreParseSuccess is returned.
   CHECK_EQ(i::PreParser::kPreParseSuccess, result);
   CHECK(log.HasError());
@@ skipping 14 matching lines @@
 
   const char* program =
       "try { } catch (e) { var foo = function () { /* first */ } }"
       "var bar = function () { /* second */ }";
 
   v8::HandleScope handles(CcTest::isolate());
   i::Handle<i::String> source = factory->NewStringFromAsciiChecked(program);
   i::GenericStringUtf16CharacterStream stream(source, 0, source->length());
   i::CompleteParserRecorder log;
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(&stream);
+  scanner.Initialize(&stream, true);
   i::Zone zone(CcTest::i_isolate()->allocator());
   i::AstValueFactory ast_value_factory(&zone,
                                        CcTest::i_isolate()->heap()->HashSeed());
   i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                          CcTest::i_isolate()->stack_guard()->real_climit());
   preparser.set_allow_lazy(true);
   i::PreParser::PreParseResult result = preparser.PreParseProgram();
   CHECK_EQ(i::PreParser::kPreParseSuccess, result);
   i::ScriptData* sd = log.GetScriptData();
   i::ParseData* pd = i::ParseData::FromCachedData(sd);
@@ skipping 31 matching lines @@
   memset(program.get(), '(', kProgramSize);
   program[kProgramSize] = '\0';
 
   uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
 
   i::Utf8ToUtf16CharacterStream stream(
       reinterpret_cast<const i::byte*>(program.get()),
       static_cast<unsigned>(kProgramSize));
   i::CompleteParserRecorder log;
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(&stream);
+  scanner.Initialize(&stream, true);
 
   i::Zone zone(CcTest::i_isolate()->allocator());
   i::AstValueFactory ast_value_factory(&zone,
                                        CcTest::i_isolate()->heap()->HashSeed());
   i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                          stack_limit);
   preparser.set_allow_lazy(true);
   i::PreParser::PreParseResult result = preparser.PreParseProgram();
   CHECK_EQ(i::PreParser::kPreParseStackOverflow, result);
 }
@@ skipping 209 matching lines @@
   }
 }
 
 #undef CHECK_EQU
 
 void TestStreamScanner(i::Utf16CharacterStream* stream,
                        i::Token::Value* expected_tokens,
                        int skip_pos = 0,  // Zero means not skipping.
                        int skip_to = 0) {
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(stream);
+  scanner.Initialize(stream, true);
 
   int i = 0;
   do {
     i::Token::Value expected = expected_tokens[i];
     i::Token::Value actual = scanner.Next();
     CHECK_EQ(i::Token::String(expected), i::Token::String(actual));
     if (scanner.location().end_pos == skip_pos) {
       scanner.SeekForward(skip_to);
     }
     i++;
@@ skipping 62 matching lines @@
   }
 }
 
 
 void TestScanRegExp(const char* re_source, const char* expected) {
   i::Utf8ToUtf16CharacterStream stream(
        reinterpret_cast<const i::byte*>(re_source),
        static_cast<unsigned>(strlen(re_source)));
   i::HandleScope scope(CcTest::i_isolate());
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(&stream);
+  scanner.Initialize(&stream, true);
 
   i::Token::Value start = scanner.peek();
   CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
   CHECK(scanner.ScanRegExpPattern(start == i::Token::ASSIGN_DIV));
   scanner.Next();  // Current token is now the regexp literal.
   i::Zone zone(CcTest::i_isolate()->allocator());
   i::AstValueFactory ast_value_factory(&zone,
                                        CcTest::i_isolate()->heap()->HashSeed());
   ast_value_factory.Internalize(CcTest::i_isolate());
   i::Handle<i::String> val =
@@ skipping 700 matching lines @@
   i::CompleteParserRecorder log;
   if (test_preparser) {
     i::Scanner scanner(isolate->unicode_cache());
     i::GenericStringUtf16CharacterStream stream(source, 0, source->length());
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->heap()->HashSeed());
     i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                            stack_limit);
     SetParserFlags(&preparser, flags);
-    scanner.Initialize(&stream);
+    scanner.Initialize(&stream, !is_module);
     i::PreParser::PreParseResult result =
         preparser.PreParseProgram(&preparser_materialized_literals, is_module);
     CHECK_EQ(i::PreParser::kPreParseSuccess, result);
   }
   bool preparse_error = log.HasError();
 
   // Parse the data
   i::FunctionLiteral* function;
   {
     i::Handle<i::Script> script = factory->NewScript(source);
@@ skipping 4101 matching lines @@
 
     i::Handle<i::Script> script = factory->NewScript(source);
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::ParseInfo info(&zone, script);
     i::Parser parser(&info);
     info.set_module();
     CHECK(!parser.Parse(&info));
   }
 }
 
+TEST(ScanHTMLEndComments) {
+  v8::V8::Initialize();
+  v8::Isolate* isolate = CcTest::isolate();
+  v8::HandleScope handles(isolate);
+
+  // Regression test. See:
+  //    http://code.google.com/p/chromium/issues/detail?id=53548
+  // Tests that --> is correctly interpreted as comment-to-end-of-line if there
+  // is only whitespace before it on the line (with comments considered as
+  // whitespace, even a multiline-comment containing a newline).
+  // This was not the case if it occurred before the first real token
+  // in the input.
+  const char* tests[] = {
+      // Before first real token.
+      "--> is eol-comment\nvar y = 37;\n",
+      "\n --> is eol-comment\nvar y = 37;\n",
+      "/* precomment */ --> is eol-comment\nvar y = 37;\n",
+      "\n/* precomment */ --> is eol-comment\nvar y = 37;\n",
+      // After first real token.
+      "var x = 42;\n--> is eol-comment\nvar y = 37;\n",
+      "var x = 42;\n/* precomment */ --> is eol-comment\nvar y = 37;\n",
+      NULL
+  };
+
+  const char* fail_tests[] = {
+      "x --> is eol-comment\nvar y = 37;\n",
+      "\"\\n\" --> is eol-comment\nvar y = 37;\n",
+      "x/* precomment */ --> is eol-comment\nvar y = 37;\n",
+      "x/* precomment\n */ --> is eol-comment\nvar y = 37;\n",
+      "var x = 42; --> is eol-comment\nvar y = 37;\n",
+      "var x = 42; /* precomment\n */ --> is eol-comment\nvar y = 37;\n",
+      NULL
+  };
+
+  // Parser/Scanner needs a stack limit.
+  CcTest::i_isolate()->stack_guard()->SetStackLimit(
+      i::GetCurrentStackPosition() - 128 * 1024);
+  uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
+  for (int i = 0; tests[i]; i++) {
+    const i::byte* source =
+        reinterpret_cast<const i::byte*>(tests[i]);
+    i::Utf8ToUtf16CharacterStream stream(source, i::StrLength(tests[i]));
+    i::CompleteParserRecorder log;
+    i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
+    scanner.Initialize(&stream, true);
+    i::Zone zone(CcTest::i_isolate()->allocator());
+    i::AstValueFactory ast_value_factory(
+        &zone, CcTest::i_isolate()->heap()->HashSeed());
+    i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
+                           stack_limit);
+    preparser.set_allow_lazy(true);
+    i::PreParser::PreParseResult result = preparser.PreParseProgram();
+    CHECK_EQ(i::PreParser::kPreParseSuccess, result);
+    CHECK(!log.HasError());
+  }
+
+  for (int i = 0; fail_tests[i]; i++) {
+    const i::byte* source =
+        reinterpret_cast<const i::byte*>(fail_tests[i]);
+    i::Utf8ToUtf16CharacterStream stream(source, i::StrLength(fail_tests[i]));
+    i::CompleteParserRecorder log;
+    i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
+    scanner.Initialize(&stream, true);
+    i::Zone zone(CcTest::i_isolate()->allocator());
+    i::AstValueFactory ast_value_factory(
+        &zone, CcTest::i_isolate()->heap()->HashSeed());
+    i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
+                           stack_limit);
+    preparser.set_allow_lazy(true);
+    i::PreParser::PreParseResult result = preparser.PreParseProgram();
+    // Even in the case of a syntax error, kPreParseSuccess is returned.
+    CHECK_EQ(i::PreParser::kPreParseSuccess, result);
+    CHECK(log.HasError());
+  }
+
+  // ES2015 B.1.3 HTML-like Comments:
+  //
+  // > The syntax and semantics of 11.4 is extended as follows except that this
+  // > extension is not allowed when parsing source code using the goal symbol
+  // > Module
+  const char* context_data[][2] = {{"", ""}, {NULL, NULL}};
+
+  RunModuleParserSyncTest(context_data, tests, kError);
+  RunModuleParserSyncTest(context_data, fail_tests, kError);
+}
+
 TEST(ModuleTopLevelFunctionDecl) {
   // clang-format off
   const char* kErrorSources[] = {
       "function f() {} function f() {}",
       "var f; function f() {}",
       "function f() {} var f;",
       "function* f() {} function* f() {}",
       "var f; function* f() {}",
       "function* f() {} var f;",
       "function f() {} function* f() {}",
@@ skipping 2031 matching lines @@
     "for (const x = 0 in {});",
     "for (let x = 0 in {});",
     NULL
   };
   // clang-format on
 
   static const ParserFlag always_flags[] = {kAllowHarmonyForIn};
   RunParserSyncTest(context_data, error_data, kError, nullptr, 0, always_flags,
                     arraysize(always_flags));
 }