Experimental parser: don't hardcode 8-bit char type.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
 //       with the distribution.
 //     * Neither the name of Google Inc. nor the names of its
 //       contributors may be used to endorse or promote products derived
 //       from this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#include "lexer.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 // FIXME: some of this is probably not needed.
 #include "allocation.h"
 #include "ast.h"
 #include "preparse-data-format.h"
 #include "preparse-data.h"
 #include "scopes.h"
 #include "preparser.h"
 #include "api.h"
 #include "ast.h"
 #include "bootstrapper.h"
 #include "char-predicates-inl.h"
 #include "codegen.h"
 #include "compiler.h"
 #include "func-name-inferrer.h"
 #include "messages.h"
 #include "parser.h"
 #include "platform.h"
 #include "preparser.h"
 #include "runtime.h"
 #include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 
+#include "experimental-scanner.h"
 
 // TODO:
 // - Run-time lexing modifications: harmony number literals, keywords depending
 //   on harmony_modules, harmony_scoping
 // - Escaping the string literals (like the baseline does)
 // - Error recovery after illegal tokens.
 
 enum Condition {
   kConditionNormal,
   kConditionDoubleQuoteString,
   kConditionSingleQuoteString,
   kConditionIdentifier,
   kConditionIdentifierIllegal,
   kConditionSingleLineComment,
   kConditionMultiLineComment,
   kConditionHtmlComment
 };
 
-#if defined(WIN32)
-
-  typedef signed char   int8_t;
-  typedef signed short  int16_t;
-  typedef signed int   int32_t;
-
-  typedef unsigned char  uint8_t;
-  typedef unsigned short uint16_t;
-  typedef unsigned int  uint32_t;
-
-#else
-
-#include <stdint.h>
-#include <unistd.h>
-
-#ifndef O_BINARY
-  #define O_BINARY 0
-#endif
-
-#endif //  defined(WIN32)
-
-#include "experimental-scanner.h"
-#include "lexer.h"
-
 using namespace v8::internal;
 
 namespace {
 
 inline int HexValue(uc32 c) {
   c -= '0';
   if (static_cast<unsigned>(c) <= 9) return c;
   c = (c | 0x20) - ('a' - '0');  // detect 0x11..0x16 and 0x31..0x36.
   if (static_cast<unsigned>(c) <= 5) return c + 10;
   return -1;
 }
 
 }
 
 #define PUSH_TOKEN(T) { send(T); SKIP(); }
 #define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); }
 #define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;}
 #define PUSH_LINE_TERMINATOR() { just_seen_line_terminator_ = true; SKIP(); }
 #define TERMINATE_ILLEGAL() { send(Token::ILLEGAL); send(Token::EOS); return 1; }
 
-#define YYCTYPE uint8_t
-
 PushScanner::PushScanner(ExperimentalScanner* sink, UnicodeCache* unicode_cache)
 : unicode_cache_(unicode_cache),
   eof_(false),
   state_(-1),
   condition_(kConditionNormal),
   limit_(NULL),
   start_(NULL),
   cursor_(NULL),
   marker_(NULL),
   real_start_(0),
@@ skipping 32 matching lines @@
 
 bool PushScanner::ValidIdentifierStart() {
   return unicode_cache_->IsIdentifierStart(ScanHexNumber(4));
 }
 
 void PushScanner::send(Token::Value token) {
   int beg = (start_ - buffer_) + real_start_;
   int end = (cursor_ - buffer_) + real_start_;
   if (FLAG_trace_lexer) {
     printf("got %s at (%d, %d): ", Token::Name(token), beg, end);
-    for (uint8_t* s = start_; s != cursor_; s++) printf("%c", (char)*s);
+    for (YYCTYPE* s = start_; s != cursor_; s++) printf("%c", (char)*s);
     printf(".\n");
   }
   just_seen_line_terminator_ = false;
   sink_->Record(token, beg, end);
 }
 
 uint32_t PushScanner::push(const void *input, int input_size) {
   if (FLAG_trace_lexer) {
     printf(
         "scanner is receiving a new data batch of length %d\n"
         "scanner continues with saved state_ = %d\n",
         input_size,
         state_);
   }
 
   //  Data source is signaling end of file when batch size
   //  is less than max_fill. This is slightly annoying because
   //  max_fill is a value that can only be known after re2c does
   //  its thing. Practically though, max_fill is never bigger than
   //  the longest keyword, so given our grammar, 32 is a safe bet.
 
-  uint8_t null[64];
+  YYCTYPE null[64];
   const int max_fill = 32;
   if (input_size < max_fill) { // FIXME: do something about this!!!
     eof_ = true;
     input = null;
     input_size = sizeof(null);
     memset(null, 0, sizeof(null));
   }
 
 
   //  When we get here, we have a partially
   //  consumed buffer_ which is in the following state_:
   //                                 last valid char    last valid buffer_ spot
   //                                 v           v
   //  +-------------------+-------------+---------------+-------------+----------------------+
   //  ^          ^       ^        ^       ^           ^
   //  buffer_       start_     marker_     cursor_    limit_         buffer_end_
   //
   //  We need to stretch the buffer_ and concatenate the new chunk of input to it
 
   size_t used = limit_ - buffer_;
   size_t needed = used + input_size;
   size_t allocated = buffer_end_ - buffer_;
   if (allocated < needed) {
     size_t limit__offset = limit_ - buffer_;
     size_t start_offset = start_ - buffer_;
     size_t marker__offset = marker_ - buffer_;
     size_t cursor__offset = cursor_ - buffer_;
 
-    buffer_ = (uint8_t*)realloc(buffer_, needed);
+    buffer_ = (YYCTYPE*)realloc(buffer_, needed);
     buffer_end_ = needed + buffer_;
 
     marker_ = marker__offset + buffer_;
     cursor_ = cursor__offset + buffer_;
     start_ = buffer_ + start_offset;
     limit_ = limit__offset + buffer_;
   }
   memcpy(limit_, input, input_size);
   limit_ += input_size;
 
@@ skipping 228 matching lines @@
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }