Streaming API: detect UTF-8 BOM.

src/scanner-character-streams.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/scanner-character-streams.h"
 
 #include "include/v8.h"
 #include "src/handles.h"
@@ skipping 214 matching lines @@
   }
   unsigned i = CopyChars(buffer_, kBufferSize, raw_data_, &raw_data_pos_,
                          raw_data_length_);
   raw_character_position_ = char_position + i;
   return i;
 }
 
 
 static const byte kUtf8MultiByteMask = 0xC0;
 static const byte kUtf8MultiByteCharFollower = 0x80;
+static const byte kUtf8MultiByteCharStart = 0xC0;
 
 
-#ifdef DEBUG
-static const byte kUtf8MultiByteCharStart = 0xC0;
 static bool IsUtf8MultiCharacterStart(byte first_byte) {
   return (first_byte & kUtf8MultiByteMask) == kUtf8MultiByteCharStart;
 }
-#endif
 
 
 static bool IsUtf8MultiCharacterFollower(byte later_byte) {
   return (later_byte & kUtf8MultiByteMask) == kUtf8MultiByteCharFollower;
 }
 
 
 // Move the cursor back to point at the preceding UTF-8 character start
 // in the buffer.
 static inline void Utf8CharacterBack(const byte* buffer, unsigned* cursor) {
@@ skipping 81 matching lines @@
       // here) and the internal parts which use unsigned. TODO(marja): make the
       // internal parts use size_t too.
       current_data_length_ =
           static_cast<unsigned>(source_stream_->GetMoreData(&current_data_));
       current_data_offset_ = 0;
       bool data_ends = current_data_length_ == 0;
 
       // A caveat: a data chunk might end with bytes from an incomplete UTF-8
       // character (the rest of the bytes will be in the next chunk).
       if (encoding_ == ScriptCompiler::StreamedSource::UTF8) {
+        if (first_chunk_) {
+          // Get rid of the byte order mark (if any).
+          if (current_data_length_ >= 3 && current_data_[0] == 0xef &&
+              current_data_[1] == 0xbb && current_data_[2] == 0xbf) {
+            current_data_offset_ = 3;
+          }
+        }
+
         HandleUtf8SplitCharacters(&data_in_buffer);
         if (!data_ends && current_data_offset_ == current_data_length_) {
           // The data stream didn't end, but we used all the data in the
           // chunk. This will only happen when the chunk was really small. We
           // don't handle the case where a UTF-8 character is split over several
           // chunks; in that case V8 won't crash, but it will be a parse error.
           delete[] current_data_;
           current_data_ = NULL;
           current_data_length_ = 0;
           current_data_offset_ = 0;
           continue;  // Request a new chunk.
         }
       }
 
       // Did the data stream end?
       if (data_ends) {
         DCHECK(utf8_split_char_buffer_length_ == 0);
         return data_in_buffer;
       }
+
+      first_chunk_ = false;
     }
 
     // Fill the buffer from current_data_.
     unsigned new_offset = 0;
     unsigned new_chars_in_buffer =
         CopyCharsHelper(buffer_ + data_in_buffer, kBufferSize - data_in_buffer,
                         current_data_ + current_data_offset_, &new_offset,
                         current_data_length_ - current_data_offset_, encoding_);
     data_in_buffer += new_chars_in_buffer;
     current_data_offset_ += new_offset;
@@ skipping 16 matching lines @@
   // Given any byte, we can always read its local environment (in both
   // directions) to find out the (possibly multi-byte) character it belongs
   // to. Single byte characters are of the form 0b0XXXXXXX. The first byte of a
   // multi-byte character is of the form 0b110XXXXX, 0b1110XXXX or
   // 0b11110XXX. The continuation bytes are of the form 0b10XXXXXX.
 
   // First check if we have leftover data from the last chunk.
   unibrow::uchar c;
   if (utf8_split_char_buffer_length_ > 0) {
     // Move the bytes which are part of the split character (which started in
-    // the previous chunk) into utf8_split_char_buffer_. Note that the
-    // continuation bytes are of the form 0b10XXXXXX, thus c >> 6 == 2.
-    while (current_data_offset_ < current_data_length_ &&
-           utf8_split_char_buffer_length_ < 4 &&
-           (c = current_data_[current_data_offset_]) >> 6 == 2) {
+    // the previous chunk) into utf8_split_char_buffer_.
+    while (
+        current_data_offset_ < current_data_length_ &&
+        utf8_split_char_buffer_length_ < 4 &&
+        IsUtf8MultiCharacterFollower(c = current_data_[current_data_offset_])) {
       utf8_split_char_buffer_[utf8_split_char_buffer_length_] = c;
       ++utf8_split_char_buffer_length_;
       ++current_data_offset_;
     }
 
     // Convert the data in utf8_split_char_buffer_.
     unsigned new_offset = 0;
     unsigned new_chars_in_buffer =
         CopyCharsHelper(buffer_ + *data_in_buffer,
                         kBufferSize - *data_in_buffer, utf8_split_char_buffer_,
                         &new_offset, utf8_split_char_buffer_length_, encoding_);
     *data_in_buffer += new_chars_in_buffer;
     // Make sure we used all the data.
     DCHECK(new_offset == utf8_split_char_buffer_length_);
     DCHECK(*data_in_buffer <= kBufferSize);
 
     utf8_split_char_buffer_length_ = 0;
   }
 
   // Move bytes which are part of an incomplete character from the end of the
   // current chunk to utf8_split_char_buffer_. They will be converted when the
   // next data chunk arrives. Note that all valid UTF-8 characters are at most 4
   // bytes long, but if the data is invalid, we can have character values bigger
   // than unibrow::Utf8::kMaxOneByteChar for more than 4 consecutive bytes.
   while (current_data_length_ > current_data_offset_ &&
-         (c = current_data_[current_data_length_ - 1]) >
-             unibrow::Utf8::kMaxOneByteChar &&
-         utf8_split_char_buffer_length_ < 4) {
+         utf8_split_char_buffer_length_ < 4 &&
+         (IsUtf8MultiCharacterFollower(
+              c = current_data_[current_data_length_ - 1]) ||
+          IsUtf8MultiCharacterStart(c))) {
     --current_data_length_;
     ++utf8_split_char_buffer_length_;
-    if (c >= (3 << 6)) {
-      // 3 << 6 = 0b11000000; this is the first byte of the multi-byte
-      // character. No need to copy the previous characters into the conversion
-      // buffer (even if they're multi-byte).
+    if (IsUtf8MultiCharacterStart(c)) {
+      // This is the first byte of the multi-byte character. No need to copy the
+      // previous characters into the conversion buffer (even if they're
+      // multi-byte).
       break;
     }
   }
   CHECK(utf8_split_char_buffer_length_ <= 4);
   for (unsigned i = 0; i < utf8_split_char_buffer_length_; ++i) {
     utf8_split_char_buffer_[i] = current_data_[current_data_length_ + i];
   }
 }
 
 
@@ skipping 11 matching lines @@
         int end_position)
     : Utf16CharacterStream(),
       source_(data),
       raw_data_(data->GetTwoByteData(start_position)) {
   buffer_cursor_ = raw_data_,
   buffer_end_ = raw_data_ + (end_position - start_position);
   pos_ = start_position;
 }
 
 } }  // namespace v8::internal