Add materialized literals for tagged templates in preparser

test/cctest/test-parsing.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1372 matching lines @@
       flags.Contains(kAllowHarmonyArrowFunctions));
   parser->set_allow_harmony_classes(flags.Contains(kAllowHarmonyClasses));
   parser->set_allow_harmony_templates(flags.Contains(kAllowHarmonyTemplates));
   parser->set_allow_harmony_sloppy(flags.Contains(kAllowHarmonySloppy));
   parser->set_allow_harmony_unicode(flags.Contains(kAllowHarmonyUnicode));
 }
 
 
 void TestParserSyncWithFlags(i::Handle<i::String> source,
                              i::EnumSet<ParserFlag> flags,
-                             ParserSyncTestResult result) {
+                             ParserSyncTestResult result,
+                             int literals = -1) {

[Dmitry Lomov (no reviews), 2014/12/10 18:49:31]

Instead of passing the expected number of literals here, could you just compare
results coming from preparser and from parser and assert that those are the
same?

   i::Isolate* isolate = CcTest::i_isolate();
   i::Factory* factory = isolate->factory();
 
   uintptr_t stack_limit = isolate->stack_guard()->real_climit();
 
   // Preparse the data.
   i::CompleteParserRecorder log;
   {
     i::Scanner scanner(isolate->unicode_cache());
     i::GenericStringUtf16CharacterStream stream(source, 0, source->length());
     i::PreParser preparser(&scanner, &log, stack_limit);
     SetParserFlags(&preparser, flags);
     scanner.Initialize(&stream);
     i::PreParser::PreParseResult result = preparser.PreParseProgram();
     CHECK_EQ(i::PreParser::kPreParseSuccess, result);
   }
 
   bool preparse_error = log.HasError();
-
+  int actual_materialized_literals;
   // Parse the data
   i::FunctionLiteral* function;
   {
     i::Handle<i::Script> script = factory->NewScript(source);
     i::CompilationInfoWithZone info(script);
     i::Parser::ParseInfo parse_info = {isolate->stack_guard()->real_climit(),
                                        isolate->heap()->HashSeed(),
                                        isolate->unicode_cache()};
     i::Parser parser(&info, &parse_info);
     SetParserFlags(&parser, flags);
     info.MarkAsGlobal();
     parser.Parse();
     function = info.function();
+    if (function) {
+      actual_materialized_literals = function->materialized_literal_count();

[caitp (gmail), 2014/12/10 18:17:49]

have to do this here because the contents of the function literal die once info
is out of scope :(

+    }
   }
 
   // Check that preparsing fails iff parsing fails.
   if (function == NULL) {
     // Extract exception from the parser.
     CHECK(isolate->has_pending_exception());
     i::Handle<i::JSObject> exception_handle(
         i::JSObject::cast(isolate->pending_exception()));
     i::Handle<i::String> message_string =
         i::Handle<i::String>::cast(i::Object::GetProperty(
@@ skipping 45 matching lines @@
         source->ToCString().get(),
         FormatMessage(log.ErrorMessageData())->ToCString().get());
     CHECK(false);
   } else if (result == kError) {
     v8::base::OS::Print(
         "Expected error on:\n"
         "\t%s\n"
         "However, parser and preparser succeeded",
         source->ToCString().get());
     CHECK(false);
+  } else if (literals > -1) {
+    if (actual_materialized_literals != literals) {
+      v8::base::OS::Print(
+        "Expected parsed function to have %d materialized literals, "
+        "but it had %d.", literals, actual_materialized_literals
+      );
+      CHECK(false);
+    }
   }
 }
 
 
 void TestParserSync(const char* source,
                     const ParserFlag* varying_flags,
                     size_t varying_flags_length,
                     ParserSyncTestResult result = kSuccessOrError,
                     const ParserFlag* always_true_flags = NULL,
                     size_t always_true_flags_length = 0,
                     const ParserFlag* always_false_flags = NULL,
-                    size_t always_false_flags_length = 0) {
+                    size_t always_false_flags_length = 0,
+                    int literals = -1) {
   i::Handle<i::String> str =
       CcTest::i_isolate()->factory()->NewStringFromAsciiChecked(source);
   for (int bits = 0; bits < (1 << varying_flags_length); bits++) {
     i::EnumSet<ParserFlag> flags;
     for (size_t flag_index = 0; flag_index < varying_flags_length;
          ++flag_index) {
       if ((bits & (1 << flag_index)) != 0) flags.Add(varying_flags[flag_index]);
     }
     for (size_t flag_index = 0; flag_index < always_true_flags_length;
          ++flag_index) {
       flags.Add(always_true_flags[flag_index]);
     }
     for (size_t flag_index = 0; flag_index < always_false_flags_length;
          ++flag_index) {
       flags.Remove(always_false_flags[flag_index]);
     }
-    TestParserSyncWithFlags(str, flags, result);
+    TestParserSyncWithFlags(str, flags, result, literals);
   }
 }
 
 
 TEST(ParserSync) {
   const char* context_data[][2] = {
     { "", "" },
     { "{", "}" },
     { "if (true) ", " else {}" },
     { "if (true) {} else ", "" },
@@ skipping 128 matching lines @@
 
 
 void RunParserSyncTest(const char* context_data[][2],
                        const char* statement_data[],
                        ParserSyncTestResult result,
                        const ParserFlag* flags = NULL,
                        int flags_len = 0,
                        const ParserFlag* always_true_flags = NULL,
                        int always_true_len = 0,
                        const ParserFlag* always_false_flags = NULL,
-                       int always_false_len = 0) {
+                       int always_false_len = 0,
+                       const int* expected_literals = NULL) {
   v8::HandleScope handles(CcTest::isolate());
   v8::Handle<v8::Context> context = v8::Context::New(CcTest::isolate());
   v8::Context::Scope context_scope(context);
 
   CcTest::i_isolate()->stack_guard()->SetStackLimit(
       i::GetCurrentStackPosition() - 128 * 1024);
 
   // Experimental feature flags should not go here; pass the flags as
   // always_true_flags if the test needs them.
   static const ParserFlag default_flags[] = {
@@ skipping 31 matching lines @@
       int kSuffixLen = i::StrLength(context_data[i][1]);
       int kProgramSize = kPrefixLen + kStatementLen + kSuffixLen;
 
       // Plug the source code pieces together.
       i::ScopedVector<char> program(kProgramSize + 1);
       int length = i::SNPrintF(program,
                                "%s%s%s",
                                context_data[i][0],
                                statement_data[j],
                                context_data[i][1]);
+      int expected_literal_count = -1;
+      if (expected_literals) {
+        expected_literal_count = expected_literals[j];
+      }
       CHECK(length == kProgramSize);
       TestParserSync(program.start(),
                      flags,
                      flags_len,
                      result,
                      always_true_flags,
                      always_true_len,
                      always_false_flags,
-                     always_false_len);
+                     always_false_len,
+                     expected_literal_count);
     }
   }
   delete[] generated_flags;
 }
 
 
 TEST(ErrorsEvalAndArguments) {
   // Tests that both preparsing and parsing produce the right kind of errors for
   // using "eval" and "arguments" as identifiers. Without the strict mode, it's
   // ok to use "eval" or "arguments" as identifiers. With the strict mode, it
@@ skipping 2723 matching lines @@
       "tag`foo${\r\n a}`",
       "tag`foo${\r a}`",
       "tag`foo${'a' in a}`",
       NULL};
   static const ParserFlag always_flags[] = {kAllowHarmonyTemplates};
   RunParserSyncTest(context_data, data, kSuccess, NULL, 0, always_flags,
                     arraysize(always_flags));
 }
 
 
+TEST(TaggedTemplateMaterializedLiterals) {
+  const char* context_data[][2] = {
+    {
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+      "'use strict';",

[caitp (gmail), 2014/12/10 18:17:49]

(lldb) p actual_materialized_literals
(int) $16 = 0
(lldb) up
frame #1: 0x00000001002ee98d cctest`TestParserSync(source=0x0000000103a04e70,
varying_flags=0x0000000103a04e60, varying_flags_length=2, result=kSuccess,
always_true_flags=0x0000000101130120, always_true_flags_length=1,
always_false_flags=0x0000000000000000, always_false_flags_length=0, literals=2)
+ 413 at test-parsing.cc:1532
   1529	         ++flag_index) {
   1530	      flags.Remove(always_false_flags[flag_index]);
   1531	    }
-> 1532	    TestParserSyncWithFlags(str, flags, result, literals);
   1533	  }
   1534	}
   1535	
(lldb) p str
(v8::internal::Handle<v8::internal::String>) $17 = {
  location_ = 0x000000010482ecc8
}
(lldb) p str->ToAsciiArray()
(char *) $18 = 0x0000000103f00050 "function test(){ 'use strict';  function
tag() {}  var a, b, c; return tag``;}"

---

So I was slightly wrong, it's not a "use strict" context, just an empty context.

+      ""
+    },
+    {
+      "function test(){ 'use strict';"
+      "  function tag() {}"
+      "  var a, b, c; return ",
+      "}"
+    },
+    {NULL, NULL}
+  };
+
+  const char* data[] = {
+    "tag``;",
+    "tag`a`;",
+    "tag`a${1}b`;",
+    "tag`a${1}b${2}c`;",
+    NULL
+  };
+
+  const int literals[] = {
+    2,
+    2,
+    2,
+    2
+  };
+
+  static const ParserFlag always_flags[] = {kAllowHarmonyTemplates};
+  RunParserSyncTest(context_data, data, kSuccess, NULL, 0, always_flags,
+                    arraysize(always_flags), NULL, 0, literals);
+}
+
+
 TEST(ScanUnterminatedTemplateLiterals) {
   const char* context_data[][2] = {{"'use strict';", ""},
                                    {"function foo(){ 'use strict';"
                                     "  var a, b, c; return ", "}"},
                                    {NULL, NULL}};
 
   const char* data[] = {
       "`no-subst-template",
       "`template-head${a}",
       "`${a}template-tail",
@@ skipping 49 matching lines @@
                     always_false_flags, arraysize(always_false_flags));
 
   const char* good_data[] = {
     "let = 1;",
     "for(let = 1;;){}",
     NULL};
   RunParserSyncTest(context_data, good_data, kSuccess, NULL, 0,
                     always_true_flags, arraysize(always_true_flags),
                     always_false_flags, arraysize(always_false_flags));
 }