Experimental parser: fix build after merge

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 200 matching lines @@
  public:
   Token::Value value;
   int beg;
   int end;
   bool is_one_byte;
   SmartArrayPointer<const uint16_t> literal;
   int literal_length;
   // The location of the latest octal position when the token was seen.
   int octal_beg;
   int octal_end;
-  TokenWithLocation(Token::Value token, Scanner* scanner) : value(token) {
+  TokenWithLocation(Token::Value token,
+                    Scanner* scanner,
+                    Handle<String> literal_string)
+      : value(token) {
     beg = scanner->location().beg_pos;
     end = scanner->location().end_pos;
     octal_beg = scanner->octal_position().beg_pos;
     octal_end = scanner->octal_position().end_pos;
     is_one_byte = false;
     literal_length = 0;
-    if (HasLiteral(token)) {
-      is_one_byte = scanner->is_literal_ascii();
-      if (scanner->is_literal_ascii()) {
-        Copy(scanner->literal_ascii_string(), &literal, &literal_length);
+    if (!literal_string.is_null()) {
+      DisallowHeapAllocation no_alloc;
+      String::FlatContent content = literal_string->GetFlatContent();
+      if (content.IsAscii()) {
+        Copy(content.ToOneByteVector(), &literal, &literal_length);
       } else {
-        Copy(scanner->literal_utf16_string(), &literal, &literal_length);
+        Copy(content.ToUC16Vector(), &literal, &literal_length);
       }
     }
   }
   void Print(bool do_compare) const {
     if (value == Token::ILLEGAL && do_compare) {
       printf("%-15s (%d)\n", Token::Name(value), beg);
       return;
     }
     printf("%-15s (%d, %d)", Token::Name(value), beg, end);
     if (literal_length > 0) {
@@ skipping 47 matching lines @@
   scanner.SetHarmonyNumericLiterals(settings.harmony_numeric_literals);
   scanner.SetHarmonyModules(settings.harmony_modules);
   scanner.SetHarmonyScoping(settings.harmony_scoping);
   ElapsedTimer timer;
   std::vector<TokenWithLocation*> tokens;
   timer.Start();
   scanner.Initialize(stream.get());
   Token::Value token;
   do {
     token = scanner.Next();
+    Handle<String> literal;
+    if (HasLiteral(token)) {
+      literal = scanner.AllocateInternalizedString(isolate);
+    }
     if (settings.print_tokens) {
-      tokens.push_back(new TokenWithLocation(token, &scanner));
-    } else if (HasLiteral(token)) {
-      if (scanner.is_literal_ascii()) {
-        scanner.literal_ascii_string();
-      } else {
-        scanner.literal_utf16_string();
-      }
+      tokens.push_back(new TokenWithLocation(token, &scanner, literal));
     }
     if (token == Token::ILLEGAL && settings.break_after_illegal) break;
   } while (token != Token::EOS);
   // Dump tokens.
   if (settings.print_tokens) {
     if (!settings.print_tokens_for_compare) {
       printf("No of tokens:\t%d\n", static_cast<int>(tokens.size()));
     }
     for (size_t i = 0; i < tokens.size(); ++i) {
       tokens[i]->Print(settings.print_tokens_for_compare);
@@ skipping 103 matching lines @@
         ++truncate_by;
       } while (can_truncate);
     }
     if (!settings.print_tokens_for_compare) {
       printf("RunTime: %.f ms\n", total_time);
     }
   }
   v8::V8::Dispose();
   return 0;
 }