Experimental scanner: keeping track of octal numbers octal escapes.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 153 matching lines @@
   BufferedUtf16CharacterStream* stream_;
 };
 
 
 struct TokenWithLocation {
   Token::Value value;
   size_t beg;
   size_t end;
   std::vector<int> literal;
   bool is_ascii;
+  // The location of the latest octal position when the token was seen.
+  int octal_beg;
+  int octal_end;
   TokenWithLocation() :
       value(Token::ILLEGAL), beg(0), end(0), is_ascii(false) { }
-  TokenWithLocation(Token::Value value, size_t beg, size_t end) :
-      value(value), beg(beg), end(end), is_ascii(false) { }
+  TokenWithLocation(Token::Value value, size_t beg, size_t end,
+                    int octal_beg, int octal_end) :
+      value(value), beg(beg), end(end), is_ascii(false), octal_beg(octal_beg),
+      octal_end(octal_end) { }
   bool operator==(const TokenWithLocation& other) {
+    // The octal_end of the baseline scanner is inconsistent between octal
+    // numbers (end = one beyond the last digit) and octal escapes (end = the
+    // last digit). Ignore that.
     return value == other.value && beg == other.beg && end == other.end &&
-           literal == other.literal && is_ascii == other.is_ascii;
+           literal == other.literal && is_ascii == other.is_ascii &&
+        octal_beg == other.octal_beg &&
+        octal_end >= other.octal_end - 1 && octal_end <= other.octal_end + 1;
   }
   bool operator!=(const TokenWithLocation& other) {
     return !(*this == other);
   }
   void Print(const char* prefix) const {
     printf("%s %11s at (%d, %d)",
            prefix, Token::Name(value),
            static_cast<int>(beg), static_cast<int>(end));
     if (literal.size() > 0) {
       for (size_t i = 0; i < literal.size(); i++) {
         printf(is_ascii ? " %02x" : " %04x", literal[i]);
       }
     }
-    printf("\n");
+    printf(" (last octal: %d %d)\n", octal_beg, octal_end);
   }
 };
 
 
 bool HasLiteral(Token::Value token) {
   return token == Token::IDENTIFIER ||
          token == Token::STRING ||
          token == Token::NUMBER;
 }
 
 
 template<typename Char>
 std::vector<int> ToStdVector(const Vector<Char>& literal) {
   std::vector<int> result;
   for (int i = 0; i < literal.length(); i++) {
     result.push_back(literal[i]);
   }
   return result;
 }
 
 
 template<typename Scanner>
 TokenWithLocation GetTokenWithLocation(Scanner *scanner, Token::Value token) {
   int beg = scanner->location().beg_pos;
   int end = scanner->location().end_pos;
-  TokenWithLocation result(token, beg, end);
+  TokenWithLocation result(token, beg, end, scanner->octal_position().beg_pos,
+                           scanner->octal_position().end_pos);
   if (HasLiteral(token)) {
     result.is_ascii = scanner->is_literal_ascii();
     if (scanner->is_literal_ascii()) {
       result.literal = ToStdVector(scanner->literal_ascii_string());
     } else {
       result.literal = ToStdVector(scanner->literal_utf16_string());
     }
   }
   return result;
 }
@@ skipping 237 matching lines @@
       if (run_experimental) {
         if (benchmark.empty()) benchmark = "Experimental";
         printf("%s(RunTime): %.f ms\n", benchmark.c_str(),
                experimental_total);
       }
     }
   }
   v8::V8::Dispose();
   return 0;
 }