| Index: src/scanner.h
|
| diff --git a/src/scanner.h b/src/scanner.h
|
| index a454750de958aef74d8196924cac22ff8e48685f..11289f6b9a0981056b49926916dcea41de847a05 100644
|
| --- a/src/scanner.h
|
| +++ b/src/scanner.h
|
| @@ -172,7 +172,12 @@ class UnicodeCache {
|
|
|
| class LiteralBuffer {
|
| public:
|
| - LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { }
|
| + // "Printable" ASCII is in the range 0x20 .. 0x7e.
|
| + enum CharacterRange { PRINTABLE_ASCII, ASCII, TWO_BYTE };
|
| +
|
| + LiteralBuffer() : character_range_(PRINTABLE_ASCII),
|
| + position_(0),
|
| + backing_store_() { }
|
|
|
| ~LiteralBuffer() {
|
| if (backing_store_.length() > 0) {
|
| @@ -182,23 +187,29 @@ class LiteralBuffer {
|
|
|
| INLINE(void AddChar(uint32_t code_unit)) {
|
| if (position_ >= backing_store_.length()) ExpandBuffer();
|
| - if (is_ascii_) {
|
| - if (code_unit <= unibrow::Latin1::kMaxChar) {
|
| - backing_store_[position_] = static_cast<byte>(code_unit);
|
| - position_ += kOneByteSize;
|
| + switch (character_range_) {
|
| + case PRINTABLE_ASCII:
|
| + if (code_unit >= 0x20 && code_unit < 0x7f) break;
|
| + character_range_ = ASCII;
|
| + case ASCII:
|
| + if (code_unit <= unibrow::Latin1::kMaxChar) break;
|
| + ConvertToUtf16();
|
| + case TWO_BYTE:
|
| + ASSERT(code_unit < 0x10000u);
|
| + *reinterpret_cast<uc16*>(&backing_store_[position_]) = code_unit;
|
| + position_ += kUC16Size;
|
| return;
|
| - }
|
| - ConvertToUtf16();
|
| }
|
| - ASSERT(code_unit < 0x10000u);
|
| - *reinterpret_cast<uc16*>(&backing_store_[position_]) = code_unit;
|
| - position_ += kUC16Size;
|
| + ASSERT(character_range_ <= ASCII);
|
| + backing_store_[position_] = static_cast<byte>(code_unit);
|
| + position_ += kOneByteSize;
|
| }
|
|
|
| - bool is_ascii() { return is_ascii_; }
|
| + inline bool is_ascii() { return character_range_ <= ASCII; }
|
| + bool is_printable_ascii() { return character_range_ == PRINTABLE_ASCII; }
|
|
|
| Vector<const uc16> utf16_literal() {
|
| - ASSERT(!is_ascii_);
|
| + ASSERT(character_range_ == TWO_BYTE);
|
| ASSERT((position_ & 0x1) == 0);
|
| return Vector<const uc16>(
|
| reinterpret_cast<const uc16*>(backing_store_.start()),
|
| @@ -206,19 +217,19 @@ class LiteralBuffer {
|
| }
|
|
|
| Vector<const char> ascii_literal() {
|
| - ASSERT(is_ascii_);
|
| + ASSERT(is_ascii());
|
| return Vector<const char>(
|
| reinterpret_cast<const char*>(backing_store_.start()),
|
| position_);
|
| }
|
|
|
| int length() {
|
| - return is_ascii_ ? position_ : (position_ >> 1);
|
| + return is_ascii() ? position_ : (position_ >> 1);
|
| }
|
|
|
| void Reset() {
|
| position_ = 0;
|
| - is_ascii_ = true;
|
| + character_range_ = PRINTABLE_ASCII;
|
| }
|
|
|
| private:
|
| @@ -240,7 +251,7 @@ class LiteralBuffer {
|
| }
|
|
|
| void ConvertToUtf16() {
|
| - ASSERT(is_ascii_);
|
| + ASSERT(character_range_ == ASCII);
|
| Vector<byte> new_store;
|
| int new_content_size = position_ * kUC16Size;
|
| if (new_content_size >= backing_store_.length()) {
|
| @@ -260,10 +271,10 @@ class LiteralBuffer {
|
| backing_store_ = new_store;
|
| }
|
| position_ = new_content_size;
|
| - is_ascii_ = false;
|
| + character_range_ = TWO_BYTE;
|
| }
|
|
|
| - bool is_ascii_;
|
| + CharacterRange character_range_;
|
| int position_;
|
| Vector<byte> backing_store_;
|
|
|
| @@ -344,6 +355,10 @@ class Scanner {
|
| ASSERT_NOT_NULL(current_.literal_chars);
|
| return current_.literal_chars->is_ascii();
|
| }
|
| + bool is_literal_printable_ascii() {
|
| + ASSERT_NOT_NULL(current_.literal_chars);
|
| + return current_.literal_chars->is_printable_ascii();
|
| + }
|
| int literal_length() const {
|
| ASSERT_NOT_NULL(current_.literal_chars);
|
| return current_.literal_chars->length();
|
|
|