Experimental parser: small fixes

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 78 matching lines @@
   // Get utf8 length.
   unsigned utf16_chars = 0;
   *is_one_byte = true;
   {
     unsigned position = 0;
     while (position < file_size) {
       uint32_t c = char_data[position];
       if (c <= unibrow::Utf8::kMaxOneByteChar) {
         position++;
       } else {
-        *is_one_byte = false;
         c =  unibrow::Utf8::CalculateValue(char_data + position,
                                            file_size - position,
                                            &position);
       }
+      if (c > unibrow::Latin1::kMaxChar) *is_one_byte = false;
       if (c > kMaxUtf16Character) {
         utf16_chars += 2;
       } else {
         utf16_chars += 1;
       }
     }
   }
   // Write new buffer out.
   uint16_t* data = new uint16_t[utf16_chars];
   unsigned position = 0;
@@ skipping 103 matching lines @@
 }
 
 
 static bool HasLiteral(Token::Value token) {
   return token == Token::IDENTIFIER ||
          token == Token::STRING ||
          token == Token::NUMBER;
 }
 
 
-template<typename Char>
-static void Copy(const Vector<Char>& literal,
-                 SmartArrayPointer<const uint16_t>* result,
-                 int* literal_length) {
-  uint16_t* data = new uint16_t[literal.length()];
-  result->Reset(data);
-  for (int i = 0; i < literal.length(); i++) {
-    data[i] = literal[i];
-  }
-  *literal_length = literal.length();
-}
-
-
 class TokenWithLocation {
  public:
-  Token::Value value;
-  int beg;
-  int end;
-  bool is_one_byte;
-  SmartArrayPointer<const uint16_t> literal;
-  int literal_length;
-  // The location of the latest octal position when the token was seen.
-  int octal_beg;
-  int octal_end;
   TokenWithLocation(Token::Value token,
                     Scanner* scanner,
                     Handle<String> literal_string)
       : value(token) {
     beg = scanner->location().beg_pos;
     end = scanner->location().end_pos;
     octal_beg = scanner->octal_position().beg_pos;
     octal_end = scanner->octal_position().end_pos;
     is_one_byte = false;
     literal_length = 0;
     if (!literal_string.is_null()) {
       DisallowHeapAllocation no_alloc;
       String::FlatContent content = literal_string->GetFlatContent();
+      literal_length = literal_string->length();
+      literal.Reset(new uint16_t[literal_length]);
       if (content.IsAscii()) {
-        Copy(content.ToOneByteVector(), &literal, &literal_length);
+        is_one_byte = true;
+        CopyChars(
+          literal.get(), content.ToOneByteVector().start(), literal_length);
       } else {
-        Copy(content.ToUC16Vector(), &literal, &literal_length);
+        CopyChars(
+          literal.get(), content.ToUC16Vector().start(), literal_length);
       }
     }
   }
   void Print(bool do_compare) const {
     if (value == Token::ILLEGAL && do_compare) {
       printf("%-15s (%d)\n", Token::Name(value), beg);
       return;
     }
     printf("%-15s (%d, %d)", Token::Name(value), beg, end);
-    if (literal_length > 0) {
+    if (literal.get() != NULL) {
       // TODO(dcarney): need some sort of checksum.
       for (int i = 0; i < literal_length; i++) {
         printf(is_one_byte ? " %02x" : " %04x", literal[i]);
       }
       printf(" (is_one_byte: %d)", is_one_byte);
     }
     if (octal_beg >= 0) {
       printf(" (last octal start: %d)", octal_beg);
     }
     printf("\n");
   }
 
  private:
+  Token::Value value;

[tfarina, 2014/03/19 17:36:50]

could you add trailing underscore to these member variables in a follow up CL
per Google C++ style guide? e.g., value_, beg_, etc...

+  int beg;
+  int end;
+  bool is_one_byte;
+  SmartArrayPointer<uint16_t> literal;
+  int literal_length;
+  // The location of the latest octal position when the token was seen.
+  int octal_beg;
+  int octal_end;
+
   DISALLOW_COPY_AND_ASSIGN(TokenWithLocation);
 };
 
 
 static TimeDelta RunLexer(const uint16_t* source,
                           const uint8_t* source_end,
                           Isolate* isolate,
                           Encoding output_encoding,
                           const LexerShellSettings& settings) {
   SmartPointer<Utf16CharacterStream> stream;
   const uint8_t* one_byte_source = reinterpret_cast<const uint8_t*>(source);
   int bytes = source_end - one_byte_source;
   switch (output_encoding) {
     case UTF8:
       stream.Reset(new Utf8ToUtf16CharacterStream(one_byte_source, bytes));
       break;
     case UTF16: {
       CHECK_EQ(0, bytes % 2);
       Handle<String> result = isolate->factory()->NewStringFromTwoByte(
-          Vector<const uint16_t>(source, bytes / 2));
+          Vector<const uint16_t>(source, bytes / 2), false);
       stream.Reset(
           new GenericStringUtf16CharacterStream(result, 0, result->length()));
       break;
     }
     case LATIN1: {
       Handle<String> result = isolate->factory()->NewStringFromOneByte(
           Vector<const uint8_t>(one_byte_source, bytes));
       stream.Reset(
           new GenericStringUtf16CharacterStream(result, 0, result->length()));
       break;
@@ skipping 143 matching lines @@
         ++truncate_by;
       } while (can_truncate);
     }
     if (!settings.print_tokens_for_compare) {
       printf("RunTime: %.f ms\n", total_time);
     }
   }
   v8::V8::Dispose();
   return 0;
 }