Handle Utf-8 BOM at beginning of an Utf-8 stream.

src/parsing/scanner-character-streams.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/parsing/scanner-character-streams.h"
 
 #include "include/v8.h"
 #include "src/globals.h"
 #include "src/handles.h"
 #include "src/objects-inl.h"
@@ skipping 268 matching lines @@
   DCHECK_EQ(buffer_start_, buffer_cursor_);
   DCHECK_LT(buffer_end_ + 1, buffer_start_ + kBufferSize);
 
   const Chunk& chunk = chunks_[current_.chunk_no];
 
   // The buffer_ is writable, but buffer_*_ members are const. So we get a
   // non-const pointer into buffer that points to the same char as buffer_end_.
   uint16_t* cursor = buffer_ + (buffer_end_ - buffer_start_);
   DCHECK_EQ(cursor, buffer_end_);
 
+  static const unibrow::uchar kUtf8Bom = 0xfeff;
+
   unibrow::Utf8::Utf8IncrementalBuffer incomplete_char =
       current_.pos.incomplete_char;
   size_t it;
   for (it = current_.pos.bytes - chunk.start.bytes;
        it < chunk.length && cursor + 1 < buffer_start_ + kBufferSize; it++) {
     unibrow::uchar t =
         unibrow::Utf8::ValueOfIncremental(chunk.data[it], &incomplete_char);
     if (t == unibrow::Utf8::kIncomplete) continue;
-    if (t <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
+    if (V8_LIKELY(t < kUtf8Bom)) {

[vogelheim, 2016/09/20 16:17:58]

1st + 3rd branch could be merged, but I wanted to keep the super common case
super simple. Not sure if that µ-optimization makes sense...

[marja, 2016/09/20 18:52:30]

I think this version is pretty readable too. 

+      *(cursor++) = static_cast<uc16>(t);  // The by most frequent case.
+    } else if (t == kUtf8Bom && current_.pos.bytes + it == 2) {
+      // BOM detected at beginning of the stream. Don't copy it.
+    } else if (t <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
       *(cursor++) = static_cast<uc16>(t);
     } else {
       *(cursor++) = unibrow::Utf16::LeadSurrogate(t);
       *(cursor++) = unibrow::Utf16::TrailSurrogate(t);
     }
   }
 
   current_.pos.bytes = chunk.start.bytes + it;
   current_.pos.chars += (cursor - buffer_end_);
   current_.pos.incomplete_char = incomplete_char;
@@ skipping 346 matching lines @@
       return new OneByteExternalStreamingStream(source_stream);
     case v8::ScriptCompiler::StreamedSource::UTF8:
       return new Utf8ExternalStreamingStream(source_stream);
   }
   UNREACHABLE();
   return nullptr;
 }
 
 }  // namespace internal
 }  // namespace v8