[scanner] Fix bom handling

src/parsing/scanner-character-streams.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/parsing/scanner-character-streams.h"
 
 #include "include/v8.h"
 #include "src/counters.h"
 #include "src/globals.h"
 #include "src/handles.h"
 #include "src/objects-inl.h"
 #include "src/parsing/scanner.h"
 #include "src/unicode-inl.h"
 
 namespace v8 {
 namespace internal {
 
+namespace {
+const unibrow::uchar kUtf8Bom = 0xfeff;
+}  // namespace
+
 // ----------------------------------------------------------------------------
 // BufferedUtf16CharacterStreams
 //
 // A buffered character stream based on a random access character
 // source (ReadBlock can be called with pos() pointing to any position,
 // even positions before the current).
 class BufferedUtf16CharacterStream : public Utf16CharacterStream {
  public:
   BufferedUtf16CharacterStream();
 
@@ skipping 228 matching lines @@
   const Chunk& chunk = chunks_[current_.chunk_no];
   DCHECK(current_.pos.bytes >= chunk.start.bytes);
 
   unibrow::Utf8::Utf8IncrementalBuffer incomplete_char =
       chunk.start.incomplete_char;
   size_t it = current_.pos.bytes - chunk.start.bytes;
   size_t chars = chunk.start.chars;
   while (it < chunk.length && chars < position) {
     unibrow::uchar t =
         unibrow::Utf8::ValueOfIncremental(chunk.data[it], &incomplete_char);
-    if (t != unibrow::Utf8::kIncomplete) {
+    if (t == kUtf8Bom && current_.pos.chars == 0) {
+      // BOM detected at beginning of the stream. Don't copy it.

[vogelheim, 2017/01/30 18:59:32]

copy -> count. This loop doesn't copy anything; it merely counts characters to
determine the correct position.

+    } else if (t != unibrow::Utf8::kIncomplete) {
       chars++;
       if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) chars++;
     }
     it++;
   }
 
   current_.pos.bytes += it;
   current_.pos.chars = chars;
   current_.pos.incomplete_char = incomplete_char;
   current_.chunk_no += (it == chunk.length);
@@ skipping 20 matching lines @@
         unibrow::Utf8::ValueOfIncrementalFinish(&current_.pos.incomplete_char);
     if (t != unibrow::Utf8::kBufferEmpty) {
       DCHECK(t < unibrow::Utf16::kMaxNonSurrogateCharCode);
       *cursor = static_cast<uc16>(t);
       buffer_end_++;
       current_.pos.chars++;
     }
     return;
   }
 
-  static const unibrow::uchar kUtf8Bom = 0xfeff;
-
   unibrow::Utf8::Utf8IncrementalBuffer incomplete_char =
       current_.pos.incomplete_char;
   size_t it;
   for (it = current_.pos.bytes - chunk.start.bytes;
        it < chunk.length && cursor + 1 < buffer_start_ + kBufferSize; it++) {
     unibrow::uchar t =
         unibrow::Utf8::ValueOfIncremental(chunk.data[it], &incomplete_char);
     if (t == unibrow::Utf8::kIncomplete) continue;
     if (V8_LIKELY(t < kUtf8Bom)) {
       *(cursor++) = static_cast<uc16>(t);  // The by most frequent case.
@@ skipping 532 matching lines @@
       return new OneByteExternalStreamingStream(source_stream, stats);
     case v8::ScriptCompiler::StreamedSource::UTF8:
       return new Utf8ExternalStreamingStream(source_stream, stats);
   }
   UNREACHABLE();
   return nullptr;
 }
 
 }  // namespace internal
 }  // namespace v8