Handle Utf-8 BOM at beginning of an Utf-8 stream.

test/cctest/parsing/test-scanner-streams.cc

 // Copyright 2016 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/factory.h"  // for i::Factory::NewExternalStringFrom*Byte
 #include "src/objects-inl.h"
 #include "src/parsing/scanner-character-streams.h"
 #include "src/parsing/scanner.h"
 #include "src/type-feedback-vector-inl.h"  // for include "src/factory.h"
 #include "test/cctest/cctest.h"
@@ skipping 91 matching lines @@
       v8::internal::ScannerStream::For(
           &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
   // Read the data without dying.
   v8::internal::uc32 c;
   do {
     c = stream->Advance();
   } while (c != v8::internal::Utf16CharacterStream::kEndOfInput);
 }
 
+TEST(Utf8StreamBOM) {
+  // Construct test string w/ UTF-8 BOM (byte order mark)
+  char data[3 + arraysize(unicode_utf8)] = {"\xef\xbb\xbf"};
+  strncpy(data + 3, unicode_utf8, arraysize(unicode_utf8));
+
+  const char* chunks[] = {data, "\0"};
+  ChunkSource chunk_source(chunks);
+  std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
+      v8::internal::ScannerStream::For(
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+
+  // Read the data without tripping over the BOM.
+  for (size_t i = 0; unicode_ucs2[i]; i++) {
+    CHECK_EQ(unicode_ucs2[i], stream->Advance());
+  }
+  CHECK_EQ(v8::internal::Utf16CharacterStream::kEndOfInput, stream->Advance());
+
+  // Make sure seek works.
+  stream->Seek(0);
+  CHECK_EQ(unicode_ucs2[0], stream->Advance());
+
+  stream->Seek(5);
+  CHECK_EQ(unicode_ucs2[5], stream->Advance());
+}
+
+TEST(Utf8SplitBOM) {
+  // Construct chunks with a BOM split into two chunks.
+  char partial_bom[] = "\xef\xbb";
+  char data[1 + arraysize(unicode_utf8)] = {"\xbf"};

[marja, 2016/09/20 18:52:30]

... how paranoid do we want to be, should there also be a test where it's all in
one-byte chunks?

[vogelheim, 2016/09/21 07:50:54]

Done.

+  strncpy(data + 1, unicode_utf8, arraysize(unicode_utf8));
+
+  const char* chunks[] = {partial_bom, data, "\0"};
+  ChunkSource chunk_source(chunks);
+  std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
+      v8::internal::ScannerStream::For(
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+
+  // Read the data without tripping over the BOM.
+  for (size_t i = 0; unicode_ucs2[i]; i++) {
+    CHECK_EQ(unicode_ucs2[i], stream->Advance());
+  }
+}
+
 TEST(Utf8ChunkBoundaries) {
   // Test utf-8 parsing at chunk boundaries.
 
   // Split the test string at each byte and pass it to the stream. This way,
   // we'll have a split at each possible boundary.
   size_t len = strlen(unicode_utf8);
   char buffer[arraysize(unicode_utf8) + 3];
   for (size_t i = 1; i < len; i++) {
     // Copy source string into buffer, splitting it at i.
     // Then add three chunks, 0..i-1, i..strlen-1, empty.
@@ skipping 233 matching lines @@
 
   // 4k large buffer.
   char buffer[4096 + 1];
   for (unsigned i = 0; i < arraysize(buffer); i++) {
     buffer[i] = static_cast<char>(i & 0x7F);
   }
   buffer[arraysize(buffer) - 1] = '\0';
   TestCharacterStreams(buffer, arraysize(buffer) - 1);
   TestCharacterStreams(buffer, arraysize(buffer) - 1, 576, 3298);
 }