Experimental parser: more correct utf8 handling

src/lexer/lexer.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 146 matching lines @@
 Token::Value LexerBase::Next() {
   has_line_terminator_before_next_ = false;
   has_multiline_comment_before_next_ = false;
   current_ = next_;
   std::swap(current_literal_, next_literal_);
   Scan();
   return current_.token;
 }
 
 
+static uint32_t Advance(const int8_t** buffer, const int8_t* end) {
+  unsigned bytes_read = 0;
+  uint32_t c = unibrow::Utf8::ValueOf(reinterpret_cast<const uint8_t*>(*buffer),
+                                      end - *buffer,
+                                      &bytes_read);
+  *buffer += bytes_read;
+  return c;
+}
+
+
+static inline uint32_t Advance(const uint8_t** buffer, const uint8_t* end) {
+  uint32_t c = **buffer;
+  (*buffer)++;
+  return c;
+}
+
+
+static inline uint32_t Advance(const uint16_t** buffer, const uint16_t* end) {
+  uint32_t c = **buffer;
+  (*buffer)++;
+  return c;
+}
+
+
 template<typename Char>
 Lexer<Char>::Lexer(UnicodeCache* unicode_cache,
                    const Char* source_ptr,
                    int length)
     : LexerBase(unicode_cache),
       isolate_(NULL),
       source_ptr_(source_ptr),
       end_position_(length),
       buffer_(source_ptr),
       buffer_end_(source_ptr + length),
@@ skipping 470 matching lines @@
 template<class Char>
 bool Lexer<Char>::CopyToLiteralBuffer(const TokenDesc& token,
                                       LiteralDesc* literal) {
   literal->buffer.Reset();
   const Char* start = NULL;
   const Char* end = NULL;
   GetStartAndEnd<Char>(buffer_, token, &start, &end);
   if (token.has_escapes) {
     for (const Char* cursor = start; cursor != end;) {
       if (*cursor != '\\') {
-        literal->buffer.AddChar(*cursor++);
+        literal->buffer.AddChar(Advance(&cursor, end));
       } else if (token.token == Token::IDENTIFIER) {
         uc32 c;
         cursor = ScanIdentifierUnicodeEscape(cursor, end, &c);
         ASSERT(cursor != NULL);
         if (cursor == NULL) return false;
         literal->buffer.AddChar(c);
       } else {
         cursor = ScanEscape(unicode_cache_, cursor, end, &literal->buffer);
         ASSERT(cursor != NULL);
         if (cursor == NULL) return false;
       }
     }
   } else {
-    // TODO(dcarney): This can only happen for utf8 strings
-    // use a helper function.
     for (const Char* cursor = start; cursor != end;) {
-        literal->buffer.AddChar(*cursor++);
+      literal->buffer.AddChar(Advance(&cursor, end));
     }
   }
   literal->SetStringFromLiteralBuffer();
   return true;
 }
 
 
 template<class Char>
 Handle<String> Lexer<Char>::AllocateInternalizedString(
     Isolate* isolate) {
@@ skipping 36 matching lines @@
   LiteralOffsetAndLength<Char>(buffer_, token, &offset, &length);
   return factory->NewSubString(source_handle_, offset, offset + length);
 }
 
 
 template class Lexer<uint8_t>;
 template class Lexer<uint16_t>;
 template class Lexer<int8_t>;
 
 } }  // v8::internal