Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(60)

Side by Side Diff: packages/utf/lib/src/utf_stream.dart

Issue 2989763002: Update charted to 0.4.8 and roll (Closed)
Patch Set: Removed Cutch from list of reviewers Created 3 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « packages/utf/lib/src/utf_16_code_unit_decoder.dart ('k') | packages/utf/lib/src/util.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 part of utf; 5 library utf.utf_stream;
6
7 import 'dart:async';
8
9 import 'constants.dart';
10 import 'util.dart';
6 11
7 // TODO(floitsch): make this transformer reusable. 12 // TODO(floitsch): make this transformer reusable.
8 abstract class _StringDecoder 13 abstract class _StringDecoder
9 implements StreamTransformer<List<int>, String>, EventSink<List<int>> { 14 implements StreamTransformer<List<int>, String>, EventSink<List<int>> {
10 List<int> _carry; 15 List<int> _carry;
11 List<int> _buffer; 16 List<int> _buffer;
12 int _replacementChar; 17 int _replacementChar;
13 18
14 EventSink<String> _outSink; 19 EventSink<String> _outSink;
15 20
16 _StringDecoder(int this._replacementChar); 21 _StringDecoder(int this._replacementChar);
17 22
18 Stream<String> bind(Stream<List<int>> stream) { 23 Stream<String> bind(Stream<List<int>> stream) {
19 return new Stream.eventTransformed( 24 return new Stream<String>.eventTransformed(stream,
20 stream,
21 (EventSink<String> sink) { 25 (EventSink<String> sink) {
22 if (_outSink != null) { 26 if (_outSink != null) {
23 throw new StateError("String decoder already used"); 27 throw new StateError("String decoder already used");
24 } 28 }
25 _outSink = sink; 29 _outSink = sink;
26 return this; 30 return this;
27 }); 31 });
28 } 32 }
29 33
30 void add(List<int> bytes) { 34 void add(List<int> bytes) {
31 try { 35 try {
32 _buffer = <int>[]; 36 _buffer = <int>[];
33 List<int> carry = _carry; 37 List<int> carry = _carry;
34 _carry = null; 38 _carry = null;
35 int pos = 0; 39 int pos = 0;
36 int available = bytes.length; 40 int available = bytes.length;
37 // If we have carry-over data, start from negative index, indicating carry 41 // If we have carry-over data, start from negative index, indicating carry
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after
110 _buffer.add(char); 114 _buffer.add(char);
111 } 115 }
112 } 116 }
113 117
114 /** 118 /**
115 * StringTransformer that decodes a stream of UTF-8 encoded bytes. 119 * StringTransformer that decodes a stream of UTF-8 encoded bytes.
116 */ 120 */
117 class Utf8DecoderTransformer extends _StringDecoder { 121 class Utf8DecoderTransformer extends _StringDecoder {
118 Utf8DecoderTransformer( 122 Utf8DecoderTransformer(
119 [int replacementChar = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) 123 [int replacementChar = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT])
120 : super(replacementChar); 124 : super(replacementChar);
121 125
122 int _processBytes(int getNext()) { 126 int _processBytes(int getNext()) {
123 int value = getNext(); 127 int value = getNext();
124 if ((value & 0xFF) != value) return -1; // Not a byte. 128 if ((value & 0xFF) != value) return -1; // Not a byte.
125 if ((value & 0x80) == 0x80) { 129 if ((value & 0x80) == 0x80) {
126 int additionalBytes; 130 int additionalBytes;
127 int min; 131 int min;
128 if ((value & 0xe0) == 0xc0) { // 110xxxxx 132 if ((value & 0xe0) == 0xc0) {
133 // 110xxxxx
129 value = value & 0x1F; 134 value = value & 0x1F;
130 additionalBytes = 1; 135 additionalBytes = 1;
131 min = 0x80; 136 min = 0x80;
132 } else if ((value & 0xf0) == 0xe0) { // 1110xxxx 137 } else if ((value & 0xf0) == 0xe0) {
138 // 1110xxxx
133 value = value & 0x0F; 139 value = value & 0x0F;
134 additionalBytes = 2; 140 additionalBytes = 2;
135 min = 0x800; 141 min = 0x800;
136 } else if ((value & 0xf8) == 0xf0) { // 11110xxx 142 } else if ((value & 0xf8) == 0xf0) {
143 // 11110xxx
137 value = value & 0x07; 144 value = value & 0x07;
138 additionalBytes = 3; 145 additionalBytes = 3;
139 min = 0x10000; 146 min = 0x10000;
140 } else if ((value & 0xfc) == 0xf8) { // 111110xx 147 } else if ((value & 0xfc) == 0xf8) {
148 // 111110xx
141 value = value & 0x03; 149 value = value & 0x03;
142 additionalBytes = 4; 150 additionalBytes = 4;
143 min = 0x200000; 151 min = 0x200000;
144 } else if ((value & 0xfe) == 0xfc) { // 1111110x 152 } else if ((value & 0xfe) == 0xfc) {
153 // 1111110x
145 value = value & 0x01; 154 value = value & 0x01;
146 additionalBytes = 5; 155 additionalBytes = 5;
147 min = 0x4000000; 156 min = 0x4000000;
148 } else { 157 } else {
149 return -1; 158 return -1;
150 } 159 }
151 for (int i = 0; i < additionalBytes; i++) { 160 for (int i = 0; i < additionalBytes; i++) {
152 int next = getNext(); 161 int next = getNext();
153 if (next == null) return 0; // Not enough chars, reset. 162 if (next == null) return 0; // Not enough chars, reset.
154 if ((next & 0xc0) != 0x80 || (next & 0xff) != next) return -1; 163 if ((next & 0xc0) != 0x80 || (next & 0xff) != next) return -1;
155 value = value << 6 | (next & 0x3f); 164 value = value << 6 | (next & 0x3f);
156 if (additionalBytes >= 3 && i == 0 && value << 12 > 0x10FFFF) { 165 if (additionalBytes >= 3 && i == 0 && value << 12 > 0x10FFFF) {
157 _addChar(-1); 166 _addChar(-1);
158 } 167 }
159 } 168 }
160 // Invalid charCode if less then minimum expected. 169 // Invalid charCode if less then minimum expected.
161 if (value < min) value = -1; 170 if (value < min) value = -1;
162 _addChar(value); 171 _addChar(value);
163 return 1 + additionalBytes; 172 return 1 + additionalBytes;
164 } 173 }
165 _addChar(value); 174 _addChar(value);
166 return 1; 175 return 1;
167 } 176 }
168 } 177 }
169 178
170
171 abstract class _StringEncoder 179 abstract class _StringEncoder
172 implements StreamTransformer<String, List<int>>, EventSink<String> { 180 implements StreamTransformer<String, List<int>>, EventSink<String> {
173
174 EventSink<List<int>> _outSink; 181 EventSink<List<int>> _outSink;
175 182
176 Stream<List<int>> bind(Stream<String> stream) { 183 Stream<List<int>> bind(Stream<String> stream) {
177 return new Stream.eventTransformed( 184 return new Stream<List<int>>.eventTransformed(stream,
178 stream,
179 (EventSink<List<int>> sink) { 185 (EventSink<List<int>> sink) {
180 if (_outSink != null) { 186 if (_outSink != null) {
181 throw new StateError("String encoder already used"); 187 throw new StateError("String encoder already used");
182 } 188 }
183 _outSink = sink; 189 _outSink = sink;
184 return this; 190 return this;
185 }); 191 });
186 } 192 }
187 193
188 void add(String data) { 194 void add(String data) {
189 _outSink.add(_processString(data)); 195 _outSink.add(_processString(data));
190 } 196 }
191 197
192 void addError(error, [StackTrace stackTrace]) { 198 void addError(error, [StackTrace stackTrace]) {
193 _outSink.addError(error, stackTrace); 199 _outSink.addError(error, stackTrace);
194 } 200 }
195 201
196 void close() { _outSink.close(); } 202 void close() {
203 _outSink.close();
204 }
197 205
198 List<int> _processString(String string); 206 List<int> _processString(String string);
199 } 207 }
200 208
201 /** 209 /**
202 * StringTransformer that UTF-8 encodes a stream of strings. 210 * StringTransformer that UTF-8 encodes a stream of strings.
203 */ 211 */
204 class Utf8EncoderTransformer extends _StringEncoder { 212 class Utf8EncoderTransformer extends _StringEncoder {
205 List<int> _processString(String string) { 213 List<int> _processString(String string) {
206 var bytes = <int>[]; 214 var bytes = <int>[];
207 int pos = 0;
208 List<int> codepoints = utf16CodeUnitsToCodepoints(string.codeUnits); 215 List<int> codepoints = utf16CodeUnitsToCodepoints(string.codeUnits);
209 int length = codepoints.length; 216 int length = codepoints.length;
210 for (int i = 0; i < length; i++) { 217 for (int i = 0; i < length; i++) {
211 int additionalBytes; 218 int additionalBytes;
212 int charCode = codepoints[i]; 219 int charCode = codepoints[i];
213 if (charCode <= 0x007F) { 220 if (charCode <= 0x007F) {
214 additionalBytes = 0; 221 additionalBytes = 0;
215 bytes.add(charCode); 222 bytes.add(charCode);
216 } else if (charCode <= 0x07FF) { 223 } else if (charCode <= 0x07FF) {
217 // 110xxxxx (xxxxx is top 5 bits). 224 // 110xxxxx (xxxxx is top 5 bits).
218 bytes.add(((charCode >> 6) & 0x1F) | 0xC0); 225 bytes.add(((charCode >> 6) & 0x1F) | 0xC0);
219 additionalBytes = 1; 226 additionalBytes = 1;
220 } else if (charCode <= 0xFFFF) { 227 } else if (charCode <= 0xFFFF) {
221 // 1110xxxx (xxxx is top 4 bits) 228 // 1110xxxx (xxxx is top 4 bits)
222 bytes.add(((charCode >> 12) & 0x0F)| 0xE0); 229 bytes.add(((charCode >> 12) & 0x0F) | 0xE0);
223 additionalBytes = 2; 230 additionalBytes = 2;
224 } else { 231 } else {
225 // 11110xxx (xxx is top 3 bits) 232 // 11110xxx (xxx is top 3 bits)
226 bytes.add(((charCode >> 18) & 0x07) | 0xF0); 233 bytes.add(((charCode >> 18) & 0x07) | 0xF0);
227 additionalBytes = 3; 234 additionalBytes = 3;
228 } 235 }
229 for (int i = additionalBytes; i > 0; i--) { 236 for (int i = additionalBytes; i > 0; i--) {
230 // 10xxxxxx (xxxxxx is next 6 bits from the top). 237 // 10xxxxxx (xxxxxx is next 6 bits from the top).
231 bytes.add(((charCode >> (6 * (i - 1))) & 0x3F) | 0x80); 238 bytes.add(((charCode >> (6 * (i - 1))) & 0x3F) | 0x80);
232 } 239 }
233 pos += additionalBytes + 1;
234 } 240 }
235 return bytes; 241 return bytes;
236 } 242 }
237 } 243 }
OLDNEW
« no previous file with comments | « packages/utf/lib/src/utf_16_code_unit_decoder.dart ('k') | packages/utf/lib/src/util.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698