| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 part of utf; | |
| 6 | |
| 7 const int _UTF8_ONE_BYTE_MAX = 0x7f; | |
| 8 const int _UTF8_TWO_BYTE_MAX = 0x7ff; | |
| 9 const int _UTF8_THREE_BYTE_MAX = 0xffff; | |
| 10 | |
| 11 const int _UTF8_LO_SIX_BIT_MASK = 0x3f; | |
| 12 | |
| 13 const int _UTF8_FIRST_BYTE_OF_TWO_BASE = 0xc0; | |
| 14 const int _UTF8_FIRST_BYTE_OF_THREE_BASE = 0xe0; | |
| 15 const int _UTF8_FIRST_BYTE_OF_FOUR_BASE = 0xf0; | |
| 16 const int _UTF8_FIRST_BYTE_OF_FIVE_BASE = 0xf8; | |
| 17 const int _UTF8_FIRST_BYTE_OF_SIX_BASE = 0xfc; | |
| 18 | |
| 19 const int _UTF8_FIRST_BYTE_OF_TWO_MASK = 0x1f; | |
| 20 const int _UTF8_FIRST_BYTE_OF_THREE_MASK = 0xf; | |
| 21 const int _UTF8_FIRST_BYTE_OF_FOUR_MASK = 0x7; | |
| 22 | |
| 23 const int _UTF8_FIRST_BYTE_BOUND_EXCL = 0xfe; | |
| 24 const int _UTF8_SUBSEQUENT_BYTE_BASE = 0x80; | |
| 25 | |
| 26 /** | |
| 27 * Decodes the UTF-8 bytes as an iterable. Thus, the consumer can only convert | |
| 28 * as much of the input as needed. Set the replacementCharacter to null to | |
| 29 * throw an ArgumentError rather than replace the bad value. | |
| 30 */ | |
| 31 IterableUtf8Decoder decodeUtf8AsIterable(List<int> bytes, [int offset = 0, | |
| 32 int length, | |
| 33 int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { | |
| 34 return new IterableUtf8Decoder(bytes, offset, length, replacementCodepoint); | |
| 35 } | |
| 36 | |
| 37 /** | |
| 38 * Produce a String from a List of UTF-8 encoded bytes. The parameters | |
| 39 * can set an offset into a list of bytes (as int), limit the length of the | |
| 40 * values to be decoded, and override the default Unicode replacement character. | |
| 41 * Set the replacementCharacter to null to throw an ArgumentError | |
| 42 * rather than replace the bad value. | |
| 43 */ | |
| 44 String decodeUtf8(List<int> bytes, [int offset = 0, int length, | |
| 45 int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { | |
| 46 return new String.fromCharCodes( | |
| 47 (new Utf8Decoder(bytes, offset, length, replacementCodepoint)) | |
| 48 .decodeRest()); | |
| 49 } | |
| 50 | |
| 51 /** | |
| 52 * Produce a sequence of UTF-8 encoded bytes from the provided string. | |
| 53 */ | |
| 54 List<int> encodeUtf8(String str) => | |
| 55 codepointsToUtf8(stringToCodepoints(str)); | |
| 56 | |
| 57 int _addToEncoding(int offset, int bytes, int value, List<int> buffer) { | |
| 58 while (bytes > 0) { | |
| 59 buffer[offset + bytes] = _UTF8_SUBSEQUENT_BYTE_BASE | | |
| 60 (value & _UTF8_LO_SIX_BIT_MASK); | |
| 61 value = value >> 6; | |
| 62 bytes--; | |
| 63 } | |
| 64 return value; | |
| 65 } | |
| 66 | |
| 67 /** | |
| 68 * Encode code points as UTF-8 code units. | |
| 69 */ | |
| 70 List<int> codepointsToUtf8( | |
| 71 List<int> codepoints, [int offset = 0, int length]) { | |
| 72 ListRange source = new ListRange(codepoints, offset, length); | |
| 73 | |
| 74 int encodedLength = 0; | |
| 75 for (int value in source) { | |
| 76 if (value < 0 || value > UNICODE_VALID_RANGE_MAX) { | |
| 77 encodedLength += 3; | |
| 78 } else if (value <= _UTF8_ONE_BYTE_MAX) { | |
| 79 encodedLength++; | |
| 80 } else if (value <= _UTF8_TWO_BYTE_MAX) { | |
| 81 encodedLength += 2; | |
| 82 } else if (value <= _UTF8_THREE_BYTE_MAX) { | |
| 83 encodedLength += 3; | |
| 84 } else if (value <= UNICODE_VALID_RANGE_MAX) { | |
| 85 encodedLength += 4; | |
| 86 } | |
| 87 } | |
| 88 | |
| 89 List<int> encoded = new List<int>(encodedLength); | |
| 90 int insertAt = 0; | |
| 91 for (int value in source) { | |
| 92 if (value < 0 || value > UNICODE_VALID_RANGE_MAX) { | |
| 93 encoded.setRange(insertAt, insertAt + 3, [0xef, 0xbf, 0xbd]); | |
| 94 insertAt += 3; | |
| 95 } else if (value <= _UTF8_ONE_BYTE_MAX) { | |
| 96 encoded[insertAt] = value; | |
| 97 insertAt++; | |
| 98 } else if (value <= _UTF8_TWO_BYTE_MAX) { | |
| 99 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_TWO_BASE | ( | |
| 100 _UTF8_FIRST_BYTE_OF_TWO_MASK & | |
| 101 _addToEncoding(insertAt, 1, value, encoded)); | |
| 102 insertAt += 2; | |
| 103 } else if (value <= _UTF8_THREE_BYTE_MAX) { | |
| 104 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_THREE_BASE | ( | |
| 105 _UTF8_FIRST_BYTE_OF_THREE_MASK & | |
| 106 _addToEncoding(insertAt, 2, value, encoded)); | |
| 107 insertAt += 3; | |
| 108 } else if (value <= UNICODE_VALID_RANGE_MAX) { | |
| 109 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_FOUR_BASE | ( | |
| 110 _UTF8_FIRST_BYTE_OF_FOUR_MASK & | |
| 111 _addToEncoding(insertAt, 3, value, encoded)); | |
| 112 insertAt += 4; | |
| 113 } | |
| 114 } | |
| 115 return encoded; | |
| 116 } | |
| 117 | |
| 118 // Because UTF-8 specifies byte order, we do not have to follow the pattern | |
| 119 // used by UTF-16 & UTF-32 regarding byte order. | |
| 120 List<int> utf8ToCodepoints( | |
| 121 List<int> utf8EncodedBytes, [int offset = 0, int length, | |
| 122 int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { | |
| 123 return new Utf8Decoder(utf8EncodedBytes, offset, length, | |
| 124 replacementCodepoint).decodeRest(); | |
| 125 } | |
| 126 | |
| 127 /** | |
| 128 * Return type of [decodeUtf8AsIterable] and variants. The Iterable type | |
| 129 * provides an iterator on demand and the iterator will only translate bytes | |
| 130 * as requested by the user of the iterator. (Note: results are not cached.) | |
| 131 */ | |
| 132 // TODO(floitsch): Consider removing the extend and switch to implements since | |
| 133 // that's cheaper to allocate. | |
| 134 class IterableUtf8Decoder extends IterableBase<int> { | |
| 135 final List<int> bytes; | |
| 136 final int offset; | |
| 137 final int length; | |
| 138 final int replacementCodepoint; | |
| 139 | |
| 140 IterableUtf8Decoder(this.bytes, [this.offset = 0, this.length = null, | |
| 141 this.replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); | |
| 142 | |
| 143 Utf8Decoder get iterator => | |
| 144 new Utf8Decoder(bytes, offset, length, replacementCodepoint); | |
| 145 } | |
| 146 | |
| 147 /** | |
| 148 * Provides an iterator of Unicode codepoints from UTF-8 encoded bytes. The | |
| 149 * parameters can set an offset into a list of bytes (as int), limit the length | |
| 150 * of the values to be decoded, and override the default Unicode replacement | |
| 151 * character. Set the replacementCharacter to null to throw an | |
| 152 * ArgumentError rather than replace the bad value. The return value | |
| 153 * from this method can be used as an Iterable (e.g. in a for-loop). | |
| 154 */ | |
| 155 class Utf8Decoder implements Iterator<int> { | |
| 156 // TODO(kevmoo): should this field be private? | |
| 157 final ListRangeIterator utf8EncodedBytesIterator; | |
| 158 final int replacementCodepoint; | |
| 159 int _current = null; | |
| 160 | |
| 161 Utf8Decoder(List<int> utf8EncodedBytes, [int offset = 0, int length, | |
| 162 this.replacementCodepoint = | |
| 163 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) : | |
| 164 utf8EncodedBytesIterator = | |
| 165 (new ListRange(utf8EncodedBytes, offset, length)).iterator; | |
| 166 | |
| 167 | |
| 168 Utf8Decoder._fromListRangeIterator(ListRange source, [ | |
| 169 this.replacementCodepoint = | |
| 170 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) : | |
| 171 utf8EncodedBytesIterator = source.iterator; | |
| 172 | |
| 173 /** Decode the remaininder of the characters in this decoder | |
| 174 * into a [List<int>]. | |
| 175 */ | |
| 176 List<int> decodeRest() { | |
| 177 List<int> codepoints = new List<int>(utf8EncodedBytesIterator.remaining); | |
| 178 int i = 0; | |
| 179 while (moveNext()) { | |
| 180 codepoints[i++] = current; | |
| 181 } | |
| 182 if (i == codepoints.length) { | |
| 183 return codepoints; | |
| 184 } else { | |
| 185 List<int> truncCodepoints = new List<int>(i); | |
| 186 truncCodepoints.setRange(0, i, codepoints); | |
| 187 return truncCodepoints; | |
| 188 } | |
| 189 } | |
| 190 | |
| 191 int get current => _current; | |
| 192 | |
| 193 bool moveNext() { | |
| 194 _current = null; | |
| 195 | |
| 196 if (!utf8EncodedBytesIterator.moveNext()) return false; | |
| 197 | |
| 198 int value = utf8EncodedBytesIterator.current; | |
| 199 int additionalBytes = 0; | |
| 200 | |
| 201 if (value < 0) { | |
| 202 if (replacementCodepoint != null) { | |
| 203 _current = replacementCodepoint; | |
| 204 return true; | |
| 205 } else { | |
| 206 throw new ArgumentError( | |
| 207 "Invalid UTF8 at ${utf8EncodedBytesIterator.position}"); | |
| 208 } | |
| 209 } else if (value <= _UTF8_ONE_BYTE_MAX) { | |
| 210 _current = value; | |
| 211 return true; | |
| 212 } else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) { | |
| 213 if (replacementCodepoint != null) { | |
| 214 _current = replacementCodepoint; | |
| 215 return true; | |
| 216 } else { | |
| 217 throw new ArgumentError( | |
| 218 "Invalid UTF8 at ${utf8EncodedBytesIterator.position}"); | |
| 219 } | |
| 220 } else if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) { | |
| 221 value -= _UTF8_FIRST_BYTE_OF_TWO_BASE; | |
| 222 additionalBytes = 1; | |
| 223 } else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) { | |
| 224 value -= _UTF8_FIRST_BYTE_OF_THREE_BASE; | |
| 225 additionalBytes = 2; | |
| 226 } else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) { | |
| 227 value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE; | |
| 228 additionalBytes = 3; | |
| 229 } else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) { | |
| 230 value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE; | |
| 231 additionalBytes = 4; | |
| 232 } else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) { | |
| 233 value -= _UTF8_FIRST_BYTE_OF_SIX_BASE; | |
| 234 additionalBytes = 5; | |
| 235 } else if (replacementCodepoint != null) { | |
| 236 _current = replacementCodepoint; | |
| 237 return true; | |
| 238 } else { | |
| 239 throw new ArgumentError( | |
| 240 "Invalid UTF8 at ${utf8EncodedBytesIterator.position}"); | |
| 241 } | |
| 242 int j = 0; | |
| 243 while (j < additionalBytes && utf8EncodedBytesIterator.moveNext()) { | |
| 244 int nextValue = utf8EncodedBytesIterator.current; | |
| 245 if (nextValue > _UTF8_ONE_BYTE_MAX && | |
| 246 nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) { | |
| 247 value = ((value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK)); | |
| 248 } else { | |
| 249 // if sequence-starting code unit, reposition cursor to start here | |
| 250 if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) { | |
| 251 utf8EncodedBytesIterator.backup(); | |
| 252 } | |
| 253 break; | |
| 254 } | |
| 255 j++; | |
| 256 } | |
| 257 bool validSequence = (j == additionalBytes && ( | |
| 258 value < UNICODE_UTF16_RESERVED_LO || | |
| 259 value > UNICODE_UTF16_RESERVED_HI)); | |
| 260 bool nonOverlong = | |
| 261 (additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) || | |
| 262 (additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) || | |
| 263 (additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX); | |
| 264 bool inRange = value <= UNICODE_VALID_RANGE_MAX; | |
| 265 if (validSequence && nonOverlong && inRange) { | |
| 266 _current = value; | |
| 267 return true; | |
| 268 } else if (replacementCodepoint != null) { | |
| 269 _current = replacementCodepoint; | |
| 270 return true; | |
| 271 } else { | |
| 272 throw new ArgumentError( | |
| 273 "Invalid UTF8 at ${utf8EncodedBytesIterator.position - j}"); | |
| 274 } | |
| 275 } | |
| 276 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2989763002:
Update charted to 0.4.8 and roll (Closed)
