Fix bad-char handling in utf-8 streaming streams. Also add test.

src/parsing/scanner-character-streams.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/parsing/scanner-character-streams.h"
 
 #include "include/v8.h"
 #include "src/globals.h"
 #include "src/handles.h"
 #include "src/objects-inl.h"
@@ skipping 268 matching lines @@
   DCHECK_EQ(buffer_start_, buffer_cursor_);
   DCHECK_LT(buffer_end_ + 1, buffer_start_ + kBufferSize);
 
   const Chunk& chunk = chunks_[current_.chunk_no];
 
   // The buffer_ is writable, but buffer_*_ members are const. So we get a
   // non-const pointer into buffer that points to the same char as buffer_end_.
   uint16_t* cursor = buffer_ + (buffer_end_ - buffer_start_);
   DCHECK_EQ(cursor, buffer_end_);
 
+  // If the current chunk is the last (empty) chunk we'll have to process
+  // any left-over, partial characters.
+  if (chunk.length == 0) {
+    unibrow::uchar t =
+        unibrow::Utf8::ValueOfIncrementalFinish(&current_.pos.incomplete_char);
+    if (t != unibrow::Utf8::kBufferEmpty) {
+      DCHECK(t < unibrow::Utf16::kMaxNonSurrogateCharCode);

[jochen (gone - plz use gerrit), 2016/10/05 16:11:56]

DCHECK_LT?

[vogelheim, 2016/10/05 16:22:23]

I tried, but couldn't get that to work. :-(

I think the issue is that kMaxNonSurr... was declared static const so that the
compile can directly use the value, and is not materialized in memory. But the
macro goo that implements DCHECK_LT eventually wants it as a const ..&
parameter. Note the reference. And taking a reference requires that the
parameter occupies memory of some sort.

The error message is:

../../src/parsing/scanner-character-streams.cc:295: error: undefined reference
to 'unibrow::Utf16::kMaxNonSurrogateCharCode'
clang: error: linker command failed with exit code 1 (use -v to see invocation)


Any ideas?

+      *cursor = static_cast<uc16>(t);
+      buffer_end_++;
+      current_.pos.chars++;
+    }
+    return;
+  }
+
   static const unibrow::uchar kUtf8Bom = 0xfeff;
 
   unibrow::Utf8::Utf8IncrementalBuffer incomplete_char =
       current_.pos.incomplete_char;
   size_t it;
   for (it = current_.pos.bytes - chunk.start.bytes;
        it < chunk.length && cursor + 1 < buffer_start_ + kBufferSize; it++) {
     unibrow::uchar t =
         unibrow::Utf8::ValueOfIncremental(chunk.data[it], &incomplete_char);
     if (t == unibrow::Utf8::kIncomplete) continue;
@@ skipping 115 matching lines @@
   if (out_of_data) return 0;
 
   // Fill the buffer, until we have at least one char (or are out of data).
   // (The embedder might give us 1-byte blocks within a utf-8 char, so we
   //  can't guarantee progress with one chunk. Thus we iterate.)
   while (!out_of_data && buffer_cursor_ == buffer_end_) {
     // At end of current data, but there might be more? Then fetch it.
     if (current_.chunk_no == chunks_.size()) {
       out_of_data = !FetchChunk();
     }
-    if (!out_of_data) FillBufferFromCurrentChunk();
+    FillBufferFromCurrentChunk();
   }
 
   DCHECK_EQ(current_.pos.chars - position, buffer_end_ - buffer_cursor_);
   return buffer_end_ - buffer_cursor_;
 }
 
 // ----------------------------------------------------------------------------
 // Chunks - helper for One- + TwoByteExternalStreamingStream
 namespace {
 
@@ skipping 225 matching lines @@
       return new OneByteExternalStreamingStream(source_stream);
     case v8::ScriptCompiler::StreamedSource::UTF8:
       return new Utf8ExternalStreamingStream(source_stream);
   }
   UNREACHABLE();
   return nullptr;
 }
 
 }  // namespace internal
 }  // namespace v8