Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(860)

Unified Diff: src/parsing/scanner-character-streams.cc

Issue 2256273002: Remove unused Utf8ToUtf16CharacterStream. (Closed) Base URL: https://chromium.googlesource.com/v8/v8.git@master
Patch Set: Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « src/parsing/scanner-character-streams.h ('k') | test/cctest/test-parsing.cc » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: src/parsing/scanner-character-streams.cc
diff --git a/src/parsing/scanner-character-streams.cc b/src/parsing/scanner-character-streams.cc
index 19c93fa3c885162c41b28deef3ac81a5f4ba9e8b..7cdef87c9c5cf429c59317c2f6176acc8e2aca6e 100644
--- a/src/parsing/scanner-character-streams.cc
+++ b/src/parsing/scanner-character-streams.cc
@@ -16,6 +16,33 @@ namespace internal {
namespace {
+size_t CopyUtf8CharsToUtf16Chars(uint16_t* dest, size_t length, const byte* src,
+ size_t* src_pos, size_t src_length) {
+ static const unibrow::uchar kMaxUtf16Character =
+ unibrow::Utf16::kMaxNonSurrogateCharCode;
+ size_t i = 0;
+ // Because of the UTF-16 lead and trail surrogates, we stop filling the buffer
+ // one character early (in the normal case), because we need to have at least
+ // two free spaces in the buffer to be sure that the next character will fit.
+ while (i < length - 1) {
+ if (*src_pos == src_length) break;
+ unibrow::uchar c = src[*src_pos];
+ if (c <= unibrow::Utf8::kMaxOneByteChar) {
+ *src_pos = *src_pos + 1;
+ } else {
+ c = unibrow::Utf8::CalculateValue(src + *src_pos, src_length - *src_pos,
+ src_pos);
+ }
+ if (c > kMaxUtf16Character) {
+ dest[i++] = unibrow::Utf16::LeadSurrogate(c);
+ dest[i++] = unibrow::Utf16::TrailSurrogate(c);
+ } else {
+ dest[i++] = static_cast<uc16>(c);
+ }
+ }
+ return i;
+}
+
size_t CopyCharsHelper(uint16_t* dest, size_t length, const uint8_t* src,
size_t* src_pos, size_t src_length,
ScriptCompiler::StreamedSource::Encoding encoding) {
@@ -24,8 +51,7 @@ size_t CopyCharsHelper(uint16_t* dest, size_t length, const uint8_t* src,
if (length == 0) return 0;
if (encoding == ScriptCompiler::StreamedSource::UTF8) {
- return v8::internal::Utf8ToUtf16CharacterStream::CopyChars(
- dest, length, src, src_pos, src_length);
+ return CopyUtf8CharsToUtf16Chars(dest, length, src, src_pos, src_length);
}
size_t to_fill = length;
@@ -175,163 +201,7 @@ size_t GenericStringUtf16CharacterStream::FillBuffer(size_t from_pos) {
// ----------------------------------------------------------------------------
-// Utf8ToUtf16CharacterStream
-Utf8ToUtf16CharacterStream::Utf8ToUtf16CharacterStream(const byte* data,
- size_t length)
- : BufferedUtf16CharacterStream(),
- raw_data_(data),
- raw_data_length_(length),
- raw_data_pos_(0),
- raw_character_position_(0) {
- ReadBlock();
-}
-
-
-Utf8ToUtf16CharacterStream::~Utf8ToUtf16CharacterStream() { }
-
-
-size_t Utf8ToUtf16CharacterStream::CopyChars(uint16_t* dest, size_t length,
- const byte* src, size_t* src_pos,
- size_t src_length) {
- static const unibrow::uchar kMaxUtf16Character =
- unibrow::Utf16::kMaxNonSurrogateCharCode;
- size_t i = 0;
- // Because of the UTF-16 lead and trail surrogates, we stop filling the buffer
- // one character early (in the normal case), because we need to have at least
- // two free spaces in the buffer to be sure that the next character will fit.
- while (i < length - 1) {
- if (*src_pos == src_length) break;
- unibrow::uchar c = src[*src_pos];
- if (c <= unibrow::Utf8::kMaxOneByteChar) {
- *src_pos = *src_pos + 1;
- } else {
- c = unibrow::Utf8::CalculateValue(src + *src_pos, src_length - *src_pos,
- src_pos);
- }
- if (c > kMaxUtf16Character) {
- dest[i++] = unibrow::Utf16::LeadSurrogate(c);
- dest[i++] = unibrow::Utf16::TrailSurrogate(c);
- } else {
- dest[i++] = static_cast<uc16>(c);
- }
- }
- return i;
-}
-
-
-size_t Utf8ToUtf16CharacterStream::BufferSeekForward(size_t delta) {
- size_t old_pos = pos_;
- size_t target_pos = pos_ + delta;
- SetRawPosition(target_pos);
- pos_ = raw_character_position_;
- ReadBlock();
- return pos_ - old_pos;
-}
-
-
-size_t Utf8ToUtf16CharacterStream::FillBuffer(size_t char_position) {
- SetRawPosition(char_position);
- if (raw_character_position_ != char_position) {
- // char_position was not a valid position in the stream (hit the end
- // while spooling to it).
- return 0u;
- }
- size_t i = CopyChars(buffer_, kBufferSize, raw_data_, &raw_data_pos_,
- raw_data_length_);
- raw_character_position_ = char_position + i;
- return i;
-}
-
-
-static const byte kUtf8MultiByteMask = 0xC0;
-static const byte kUtf8MultiByteCharFollower = 0x80;
-
-
-#ifdef DEBUG
-static const byte kUtf8MultiByteCharStart = 0xC0;
-static bool IsUtf8MultiCharacterStart(byte first_byte) {
- return (first_byte & kUtf8MultiByteMask) == kUtf8MultiByteCharStart;
-}
-#endif
-
-
-static bool IsUtf8MultiCharacterFollower(byte later_byte) {
- return (later_byte & kUtf8MultiByteMask) == kUtf8MultiByteCharFollower;
-}
-
-
-// Move the cursor back to point at the preceding UTF-8 character start
-// in the buffer.
-static inline void Utf8CharacterBack(const byte* buffer, size_t* cursor) {
- byte character = buffer[--*cursor];
- if (character > unibrow::Utf8::kMaxOneByteChar) {
- DCHECK(IsUtf8MultiCharacterFollower(character));
- // Last byte of a multi-byte character encoding. Step backwards until
- // pointing to the first byte of the encoding, recognized by having the
- // top two bits set.
- while (IsUtf8MultiCharacterFollower(buffer[--*cursor])) { }
- DCHECK(IsUtf8MultiCharacterStart(buffer[*cursor]));
- }
-}
-
-
-// Move the cursor forward to point at the next following UTF-8 character start
-// in the buffer.
-static inline void Utf8CharacterForward(const byte* buffer, size_t* cursor) {
- byte character = buffer[(*cursor)++];
- if (character > unibrow::Utf8::kMaxOneByteChar) {
- // First character of a multi-byte character encoding.
- // The number of most-significant one-bits determines the length of the
- // encoding:
- // 110..... - (0xCx, 0xDx) one additional byte (minimum).
- // 1110.... - (0xEx) two additional bytes.
- // 11110... - (0xFx) three additional bytes (maximum).
- DCHECK(IsUtf8MultiCharacterStart(character));
- // Additional bytes is:
- // 1 if value in range 0xC0 .. 0xDF.
- // 2 if value in range 0xE0 .. 0xEF.
- // 3 if value in range 0xF0 .. 0xF7.
- // Encode that in a single value.
- size_t additional_bytes =
- ((0x3211u) >> (((character - 0xC0) >> 2) & 0xC)) & 0x03;
- *cursor += additional_bytes;
- DCHECK(!IsUtf8MultiCharacterFollower(buffer[1 + additional_bytes]));
- }
-}
-
-
-// This can't set a raw position between two surrogate pairs, since there
-// is no position in the UTF8 stream that corresponds to that. This assumes
-// that the surrogate pair is correctly coded as a 4 byte UTF-8 sequence. If
-// it is illegally coded as two 3 byte sequences then there is no problem here.
-void Utf8ToUtf16CharacterStream::SetRawPosition(size_t target_position) {
- if (raw_character_position_ > target_position) {
- // Spool backwards in utf8 buffer.
- do {
- size_t old_pos = raw_data_pos_;
- Utf8CharacterBack(raw_data_, &raw_data_pos_);
- raw_character_position_--;
- DCHECK(old_pos - raw_data_pos_ <= 4);
- // Step back over both code units for surrogate pairs.
- if (old_pos - raw_data_pos_ == 4) raw_character_position_--;
- } while (raw_character_position_ > target_position);
- // No surrogate pair splitting.
- DCHECK(raw_character_position_ == target_position);
- return;
- }
- // Spool forwards in the utf8 buffer.
- while (raw_character_position_ < target_position) {
- if (raw_data_pos_ == raw_data_length_) return;
- size_t old_pos = raw_data_pos_;
- Utf8CharacterForward(raw_data_, &raw_data_pos_);
- raw_character_position_++;
- DCHECK(raw_data_pos_ - old_pos <= 4);
- if (raw_data_pos_ - old_pos == 4) raw_character_position_++;
- }
- // No surrogate pair splitting.
- DCHECK(raw_character_position_ == target_position);
-}
-
+// ExternalStreamingStream
size_t ExternalStreamingStream::FillBuffer(size_t position) {
// Ignore "position" which is the position in the decoded data. Instead,
@@ -599,6 +469,16 @@ ExternalOneByteStringUtf16CharacterStream::
pos_ = start_position;
}
+ExternalOneByteStringUtf16CharacterStream::
+ ExternalOneByteStringUtf16CharacterStream(const char* data, size_t length)
+ : raw_data_(reinterpret_cast<const uint8_t*>(data)),
+ length_(length),
+ bookmark_(kNoBookmark) {}
+
+ExternalOneByteStringUtf16CharacterStream::
+ ExternalOneByteStringUtf16CharacterStream(const char* data)
+ : ExternalOneByteStringUtf16CharacterStream(data, strlen(data)) {}
+
bool ExternalOneByteStringUtf16CharacterStream::SetBookmark() {
bookmark_ = pos_;
return true;
« no previous file with comments | « src/parsing/scanner-character-streams.h ('k') | test/cctest/test-parsing.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698