Remove unused Utf8ToUtf16CharacterStream.

test/cctest/test-parsing.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 46 matching lines @@
 
   static const KeywordToken keywords[] = {
 #define KEYWORD(t, s, d) { s, i::Token::t },
       TOKEN_LIST(IGNORE_TOKEN, KEYWORD)
 #undef KEYWORD
       { NULL, i::Token::IDENTIFIER }
   };
 
   KeywordToken key_token;
   i::UnicodeCache unicode_cache;
-  i::byte buffer[32];
+  char buffer[32];
   for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) {
-    const i::byte* keyword =
-        reinterpret_cast<const i::byte*>(key_token.keyword);
-    int length = i::StrLength(key_token.keyword);
+    const char* keyword = key_token.keyword;
+    size_t length = strlen(key_token.keyword);
     CHECK(static_cast<int>(sizeof(buffer)) >= length);
     {
-      i::Utf8ToUtf16CharacterStream stream(keyword, length);
+      i::ExternalOneByteStringUtf16CharacterStream stream(keyword, length);
       i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(key_token.token, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Removing characters will make keyword matching fail.
     {
-      i::Utf8ToUtf16CharacterStream stream(keyword, length - 1);
+      i::ExternalOneByteStringUtf16CharacterStream stream(keyword, length - 1);
       i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Adding characters will make keyword matching fail.
     static const char chars_to_append[] = { 'z', '0', '_' };
     for (int j = 0; j < static_cast<int>(arraysize(chars_to_append)); ++j) {
       i::MemMove(buffer, keyword, length);
       buffer[length] = chars_to_append[j];
-      i::Utf8ToUtf16CharacterStream stream(buffer, length + 1);
+      i::ExternalOneByteStringUtf16CharacterStream stream(buffer, length + 1);
       i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Replacing characters will make keyword matching fail.
     {
       i::MemMove(buffer, keyword, length);
       buffer[length - 1] = '_';
-      i::Utf8ToUtf16CharacterStream stream(buffer, length);
+      i::ExternalOneByteStringUtf16CharacterStream stream(buffer, length);
       i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
   }
 }
 
 
 TEST(ScanHTMLEndComments) {
@@ skipping 44 matching lines @@
       "var x = 42; --> is eol-comment\nvar y = 37;\n",
       NULL
   };
   // clang-format on
 
   // Parser/Scanner needs a stack limit.
   CcTest::i_isolate()->stack_guard()->SetStackLimit(
       i::GetCurrentStackPosition() - 128 * 1024);
   uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
   for (int i = 0; tests[i]; i++) {
-    const i::byte* source =
-        reinterpret_cast<const i::byte*>(tests[i]);
-    i::Utf8ToUtf16CharacterStream stream(source, i::StrLength(tests[i]));
+    const char* source = tests[i];
+    i::ExternalOneByteStringUtf16CharacterStream stream(source);
     i::CompleteParserRecorder log;
     i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
     scanner.Initialize(&stream);
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->heap()->HashSeed());
     i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                            stack_limit);
     preparser.set_allow_lazy(true);
     i::PreParser::PreParseResult result = preparser.PreParseProgram();
     CHECK_EQ(i::PreParser::kPreParseSuccess, result);
     CHECK(!log.HasError());
   }
 
   for (int i = 0; fail_tests[i]; i++) {
-    const i::byte* source =
-        reinterpret_cast<const i::byte*>(fail_tests[i]);
-    i::Utf8ToUtf16CharacterStream stream(source, i::StrLength(fail_tests[i]));
+    const char* source = fail_tests[i];
+    i::ExternalOneByteStringUtf16CharacterStream stream(source);
     i::CompleteParserRecorder log;
     i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
     scanner.Initialize(&stream);
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->heap()->HashSeed());
     i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                            stack_limit);
     preparser.set_allow_lazy(true);
     i::PreParser::PreParseResult result = preparser.PreParseProgram();
@@ skipping 137 matching lines @@
       "function foo(x, y) { return x + y; }",
       "%ArgleBargle(glop);",
       "var x = new new Function('this.x = 42');",
       "var f = (x, y) => x + y;",
       NULL
   };
 
   uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     const char* program = programs[i];
-    i::Utf8ToUtf16CharacterStream stream(
-        reinterpret_cast<const i::byte*>(program),
-        static_cast<unsigned>(strlen(program)));
+    i::ExternalOneByteStringUtf16CharacterStream stream(program);
     i::CompleteParserRecorder log;
     i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
     scanner.Initialize(&stream);
 
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->heap()->HashSeed());
     i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                            stack_limit);
     preparser.set_allow_lazy(true);
@@ skipping 13 matching lines @@
 
   const char* programs[] = {
       "%ArgleBargle(glop);",
       "var x = %_IsSmi(42);",
       NULL
   };
 
   uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     const char* program = programs[i];
-    i::Utf8ToUtf16CharacterStream stream(
-        reinterpret_cast<const i::byte*>(program),
-        static_cast<unsigned>(strlen(program)));
+    i::ExternalOneByteStringUtf16CharacterStream stream(program);
     i::CompleteParserRecorder log;
     i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
     scanner.Initialize(&stream);
 
     // Preparser defaults to disallowing natives syntax.
     i::Zone zone(CcTest::i_isolate()->allocator());
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->heap()->HashSeed());
     i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                            stack_limit);
@@ skipping 48 matching lines @@
   isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() -
                                         128 * 1024);
 
   const char* program = "var x = 'something';\n"
                         "escape: function() {}";
   // Fails parsing expecting an identifier after "function".
   // Before fix, didn't check *ok after Expect(Token::Identifier, ok),
   // and then used the invalid currently scanned literal. This always
   // failed in debug mode, and sometimes crashed in release mode.
 
-  i::Utf8ToUtf16CharacterStream stream(
-      reinterpret_cast<const i::byte*>(program),
-      static_cast<unsigned>(strlen(program)));
+  i::ExternalOneByteStringUtf16CharacterStream stream(program);
   i::CompleteParserRecorder log;
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
   scanner.Initialize(&stream);
   i::Zone zone(CcTest::i_isolate()->allocator());
   i::AstValueFactory ast_value_factory(&zone,
                                        CcTest::i_isolate()->heap()->HashSeed());
   i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                          CcTest::i_isolate()->stack_guard()->real_climit());
   preparser.set_allow_lazy(true);
   i::PreParser::PreParseResult result = preparser.PreParseProgram();
@@ skipping 63 matching lines @@
   CcTest::i_isolate()->stack_guard()->SetStackLimit(
       i::GetCurrentStackPosition() - 128 * 1024);
 
   size_t kProgramSize = 1024 * 1024;
   std::unique_ptr<char[]> program(i::NewArray<char>(kProgramSize + 1));
   memset(program.get(), '(', kProgramSize);
   program[kProgramSize] = '\0';
 
   uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
 
-  i::Utf8ToUtf16CharacterStream stream(
-      reinterpret_cast<const i::byte*>(program.get()),
-      static_cast<unsigned>(kProgramSize));
+  i::ExternalOneByteStringUtf16CharacterStream stream(program.get(),
+                                                      kProgramSize);
   i::CompleteParserRecorder log;
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
   scanner.Initialize(&stream);
 
   i::Zone zone(CcTest::i_isolate()->allocator());
   i::AstValueFactory ast_value_factory(&zone,
                                        CcTest::i_isolate()->heap()->HashSeed());
   i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
                          stack_limit);
   preparser.set_allow_lazy(true);
@@ skipping 47 matching lines @@
       factory->NewExternalStringFromOneByte(&one_byte_resource)
           .ToHandleChecked());
 
   i::ExternalTwoByteStringUtf16CharacterStream uc16_stream(
       i::Handle<i::ExternalTwoByteString>::cast(uc16_string), start, end);
   i::ExternalOneByteStringUtf16CharacterStream one_byte_stream(
       i::Handle<i::ExternalOneByteString>::cast(ext_one_byte_string), start,
       end);
   i::GenericStringUtf16CharacterStream string_stream(one_byte_string, start,
                                                      end);
-  i::Utf8ToUtf16CharacterStream utf8_stream(
-      reinterpret_cast<const i::byte*>(one_byte_source), end);
+  i::ExternalOneByteStringUtf16CharacterStream utf8_stream(one_byte_source,
+                                                           end);
   utf8_stream.SeekForward(start);
 
   unsigned i = start;
   while (i < end) {
     // Read streams one char at a time
     CHECK_EQU(i, uc16_stream.pos());
     CHECK_EQU(i, string_stream.pos());
     CHECK_EQU(i, utf8_stream.pos());
     CHECK_EQU(i, one_byte_stream.pos());
     int32_t c0 = one_byte_source[i];
@@ skipping 86 matching lines @@
   int32_t c1 = uc16_stream.Advance();
   int32_t c2 = string_stream.Advance();
   int32_t c3 = utf8_stream.Advance();
   int32_t c4 = one_byte_stream.Advance();
   CHECK_LT(c1, 0);
   CHECK_LT(c2, 0);
   CHECK_LT(c3, 0);
   CHECK_LT(c4, 0);
 }
 
+#undef CHECK_EQU
 
 TEST(CharacterStreams) {
   v8::Isolate* isolate = CcTest::isolate();
   v8::HandleScope handles(isolate);
   v8::Local<v8::Context> context = v8::Context::New(isolate);
   v8::Context::Scope context_scope(context);
 
   TestCharacterStream("abc\0\n\r\x7f", 7);
   static const unsigned kBigStringSize = 4096;
   char buffer[kBigStringSize + 1];
   for (unsigned i = 0; i < kBigStringSize; i++) {
     buffer[i] = static_cast<char>(i & 0x7f);
   }
   TestCharacterStream(buffer, kBigStringSize);
 
   TestCharacterStream(buffer, kBigStringSize, 576, 3298);
 
   TestCharacterStream("\0", 1);
   TestCharacterStream("", 0);
 }
 
 
-TEST(Utf8CharacterStream) {
-  static const unsigned kMaxUC16CharU = unibrow::Utf8::kMaxThreeByteChar;
-  static const int kMaxUC16Char = static_cast<int>(kMaxUC16CharU);
-
-  static const int kAllUtf8CharsSize =
-      (unibrow::Utf8::kMaxOneByteChar + 1) +
-      (unibrow::Utf8::kMaxTwoByteChar - unibrow::Utf8::kMaxOneByteChar) * 2 +
-      (unibrow::Utf8::kMaxThreeByteChar - unibrow::Utf8::kMaxTwoByteChar) * 3;
-  static const unsigned kAllUtf8CharsSizeU =
-      static_cast<unsigned>(kAllUtf8CharsSize);
-
-  char buffer[kAllUtf8CharsSizeU];
-  unsigned cursor = 0;
-  for (int i = 0; i <= kMaxUC16Char; i++) {
-    cursor += unibrow::Utf8::Encode(buffer + cursor, i,
-                                    unibrow::Utf16::kNoPreviousCharacter, true);
-  }
-  CHECK(cursor == kAllUtf8CharsSizeU);
-
-  i::Utf8ToUtf16CharacterStream stream(reinterpret_cast<const i::byte*>(buffer),
-                                       kAllUtf8CharsSizeU);
-  int32_t bad = unibrow::Utf8::kBadChar;
-  for (int i = 0; i <= kMaxUC16Char; i++) {
-    CHECK_EQU(i, stream.pos());
-    int32_t c = stream.Advance();
-    if (i >= 0xd800 && i <= 0xdfff) {
-      CHECK_EQ(bad, c);
-    } else {
-      CHECK_EQ(i, c);
-    }
-    CHECK_EQU(i + 1, stream.pos());
-  }
-  for (int i = kMaxUC16Char; i >= 0; i--) {
-    CHECK_EQU(i + 1, stream.pos());
-    stream.PushBack(i);
-    CHECK_EQU(i, stream.pos());
-  }
-  int i = 0;
-  while (stream.pos() < kMaxUC16CharU) {
-    CHECK_EQU(i, stream.pos());
-    int progress = static_cast<int>(stream.SeekForward(12));
-    i += progress;
-    int32_t c = stream.Advance();
-    if (i >= 0xd800 && i <= 0xdfff) {
-      CHECK_EQ(bad, c);
-    } else if (i <= kMaxUC16Char) {
-      CHECK_EQ(i, c);
-    } else {
-      CHECK_EQ(-1, c);
-    }
-    i += 1;
-    CHECK_EQU(i, stream.pos());
-  }
-}
-
-#undef CHECK_EQU
-
 void TestStreamScanner(i::Utf16CharacterStream* stream,
                        i::Token::Value* expected_tokens,
                        int skip_pos = 0,  // Zero means not skipping.
                        int skip_to = 0) {
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
   scanner.Initialize(stream);
 
   int i = 0;
   do {
     i::Token::Value expected = expected_tokens[i];
     i::Token::Value actual = scanner.Next();
     CHECK_EQ(i::Token::String(expected), i::Token::String(actual));
     if (scanner.location().end_pos == skip_pos) {
       scanner.SeekForward(skip_to);
     }
     i++;
   } while (expected_tokens[i] != i::Token::ILLEGAL);
 }
 
 
 TEST(StreamScanner) {
   v8::V8::Initialize();
 
   const char* str1 = "{ foo get for : */ <- \n\n /*foo*/ bib";
-  i::Utf8ToUtf16CharacterStream stream1(reinterpret_cast<const i::byte*>(str1),
-                                        static_cast<unsigned>(strlen(str1)));
+  i::ExternalOneByteStringUtf16CharacterStream stream1(str1);
   i::Token::Value expectations1[] = {
       i::Token::LBRACE,
       i::Token::IDENTIFIER,
       i::Token::IDENTIFIER,
       i::Token::FOR,
       i::Token::COLON,
       i::Token::MUL,
       i::Token::DIV,
       i::Token::LT,
       i::Token::SUB,
       i::Token::IDENTIFIER,
       i::Token::EOS,
       i::Token::ILLEGAL
   };
   TestStreamScanner(&stream1, expectations1, 0, 0);
 
   const char* str2 = "case default const {THIS\nPART\nSKIPPED} do";
-  i::Utf8ToUtf16CharacterStream stream2(reinterpret_cast<const i::byte*>(str2),
-                                        static_cast<unsigned>(strlen(str2)));
+  i::ExternalOneByteStringUtf16CharacterStream stream2(str2);
   i::Token::Value expectations2[] = {
       i::Token::CASE,
       i::Token::DEFAULT,
       i::Token::CONST,
       i::Token::LBRACE,
       // Skipped part here
       i::Token::RBRACE,
       i::Token::DO,
       i::Token::EOS,
       i::Token::ILLEGAL
   };
   CHECK_EQ('{', str2[19]);
   CHECK_EQ('}', str2[37]);
   TestStreamScanner(&stream2, expectations2, 20, 37);
 
   const char* str3 = "{}}}}";
   i::Token::Value expectations3[] = {
       i::Token::LBRACE,
       i::Token::RBRACE,
       i::Token::RBRACE,
       i::Token::RBRACE,
       i::Token::RBRACE,
       i::Token::EOS,
       i::Token::ILLEGAL
   };
   // Skip zero-four RBRACEs.
   for (int i = 0; i <= 4; i++) {
      expectations3[6 - i] = i::Token::ILLEGAL;
      expectations3[5 - i] = i::Token::EOS;
-     i::Utf8ToUtf16CharacterStream stream3(
-         reinterpret_cast<const i::byte*>(str3),
-         static_cast<unsigned>(strlen(str3)));
+     i::ExternalOneByteStringUtf16CharacterStream stream3(str3);
      TestStreamScanner(&stream3, expectations3, 1, 1 + i);
   }
 }
 
 
 void TestScanRegExp(const char* re_source, const char* expected) {
-  i::Utf8ToUtf16CharacterStream stream(
-       reinterpret_cast<const i::byte*>(re_source),
-       static_cast<unsigned>(strlen(re_source)));
+  i::ExternalOneByteStringUtf16CharacterStream stream(re_source);
   i::HandleScope scope(CcTest::i_isolate());
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
   scanner.Initialize(&stream);
 
   i::Token::Value start = scanner.peek();
   CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
   CHECK(scanner.ScanRegExpPattern());
   scanner.Next();  // Current token is now the regexp literal.
   i::Zone zone(CcTest::i_isolate()->allocator());
   i::AstValueFactory ast_value_factory(&zone,
@@ skipping 7249 matching lines @@
     "(a,);",
     "(a,b,c,);",
     NULL
   };
   // clang-format on
 
   static const ParserFlag always_flags[] = {kAllowHarmonyTrailingCommas};
   RunParserSyncTest(context_data, data, kError, NULL, 0, always_flags,
                     arraysize(always_flags));
 }