OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 #include "platform/assert.h" | 5 #include "platform/assert.h" |
6 #include "vm/os.h" | 6 #include "vm/os.h" |
7 #include "vm/scanner.h" | 7 #include "vm/scanner.h" |
8 #include "vm/token.h" | 8 #include "vm/token.h" |
9 #include "vm/unit_test.h" | 9 #include "vm/unit_test.h" |
10 | 10 |
(...skipping 22 matching lines...) Expand all Loading... |
33 printf("%d tokens in stream.\n", token_index); | 33 printf("%d tokens in stream.\n", token_index); |
34 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); | 34 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); |
35 } | 35 } |
36 | 36 |
37 | 37 |
38 void CheckKind(const Scanner::GrowableTokenStream &token_stream, | 38 void CheckKind(const Scanner::GrowableTokenStream &token_stream, |
39 int index, | 39 int index, |
40 Token::Kind kind) { | 40 Token::Kind kind) { |
41 if (token_stream[index].kind != kind) { | 41 if (token_stream[index].kind != kind) { |
42 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, | 42 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, |
43 Token::Name(token_stream[index].kind), Token::Name(kind)); | 43 Token::Name(kind), Token::Name(token_stream[index].kind)); |
44 } | 44 } |
45 EXPECT_EQ(kind, token_stream[index].kind); | 45 EXPECT_EQ(kind, token_stream[index].kind); |
46 } | 46 } |
47 | 47 |
48 | 48 |
49 void CheckLiteral(const Scanner::GrowableTokenStream& token_stream, | 49 void CheckLiteral(const Scanner::GrowableTokenStream& token_stream, |
50 int index, | 50 int index, |
51 const char* literal) { | 51 const char* literal) { |
52 if (token_stream[index].literal == NULL) { | 52 if (token_stream[index].literal == NULL) { |
53 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", | 53 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", |
(...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
163 EXPECT_EQ('"', litchars[0]); | 163 EXPECT_EQ('"', litchars[0]); |
164 EXPECT_EQ(' ', litchars[1]); | 164 EXPECT_EQ(' ', litchars[1]); |
165 EXPECT_EQ('\\', litchars[2]); | 165 EXPECT_EQ('\\', litchars[2]); |
166 EXPECT_EQ('\n', litchars[4]); | 166 EXPECT_EQ('\n', litchars[4]); |
167 EXPECT_EQ('\r', litchars[5]); | 167 EXPECT_EQ('\r', litchars[5]); |
168 EXPECT_EQ('\t', litchars[6]); | 168 EXPECT_EQ('\t', litchars[6]); |
169 EXPECT_EQ('\'', litchars[8]); | 169 EXPECT_EQ('\'', litchars[8]); |
170 } | 170 } |
171 | 171 |
172 | 172 |
| 173 void InvalidStringEscapes() { |
| 174 const Scanner::GrowableTokenStream& high_start_4 = |
| 175 Scan("\"\\uD800\""); |
| 176 EXPECT_EQ(2, high_start_4.length()); |
| 177 CheckKind(high_start_4, 0, Token::kERROR); |
| 178 EXPECT(high_start_4[0].literal->Equals("invalid code point")); |
| 179 CheckKind(high_start_4, 1, Token::kEOS); |
| 180 |
| 181 const Scanner::GrowableTokenStream& high_start_seq = |
| 182 Scan("\"\\u{D800}\""); |
| 183 EXPECT_EQ(2, high_start_seq.length()); |
| 184 CheckKind(high_start_seq, 0, Token::kERROR); |
| 185 EXPECT(high_start_seq[0].literal->Equals("invalid code point")); |
| 186 CheckKind(high_start_seq, 1, Token::kEOS); |
| 187 |
| 188 const Scanner::GrowableTokenStream& high_end_4 = |
| 189 Scan("\"\\uDBFF\""); |
| 190 EXPECT_EQ(2, high_end_4.length()); |
| 191 CheckKind(high_end_4, 0, Token::kERROR); |
| 192 EXPECT(high_end_4[0].literal->Equals("invalid code point")); |
| 193 CheckKind(high_end_4, 1, Token::kEOS); |
| 194 |
| 195 const Scanner::GrowableTokenStream& high_end_seq = |
| 196 Scan("\"\\u{DBFF}\""); |
| 197 EXPECT_EQ(2, high_end_seq.length()); |
| 198 CheckKind(high_end_seq, 0, Token::kERROR); |
| 199 EXPECT(high_end_seq[0].literal->Equals("invalid code point")); |
| 200 CheckKind(high_end_seq, 1, Token::kEOS); |
| 201 |
| 202 const Scanner::GrowableTokenStream& low_start_4 = |
| 203 Scan("\"\\uDC00\""); |
| 204 EXPECT_EQ(2, low_start_4.length()); |
| 205 CheckKind(low_start_4, 0, Token::kERROR); |
| 206 EXPECT(low_start_4[0].literal->Equals("invalid code point")); |
| 207 CheckKind(low_start_4, 1, Token::kEOS); |
| 208 |
| 209 const Scanner::GrowableTokenStream& low_start_seq = |
| 210 Scan("\"\\u{DC00}\""); |
| 211 EXPECT_EQ(2, low_start_seq.length()); |
| 212 CheckKind(low_start_seq, 0, Token::kERROR); |
| 213 EXPECT(low_start_seq[0].literal->Equals("invalid code point")); |
| 214 CheckKind(low_start_seq, 1, Token::kEOS); |
| 215 |
| 216 const Scanner::GrowableTokenStream& low_end_4 = |
| 217 Scan("\"\\uDFFF\""); |
| 218 EXPECT_EQ(2, low_end_4.length()); |
| 219 CheckKind(low_end_4, 0, Token::kERROR); |
| 220 EXPECT(low_end_4[0].literal->Equals("invalid code point")); |
| 221 CheckKind(low_end_4, 1, Token::kEOS); |
| 222 |
| 223 const Scanner::GrowableTokenStream& low_end_seq = |
| 224 Scan("\"\\u{DFFF}\""); |
| 225 EXPECT_EQ(2, low_end_seq.length()); |
| 226 CheckKind(low_end_seq, 0, Token::kERROR); |
| 227 EXPECT(low_end_seq[0].literal->Equals("invalid code point")); |
| 228 CheckKind(low_end_seq, 1, Token::kEOS); |
| 229 |
| 230 const Scanner::GrowableTokenStream& out_of_range_low = |
| 231 Scan("\"\\u{110000}\""); |
| 232 EXPECT_EQ(2, out_of_range_low.length()); |
| 233 CheckKind(out_of_range_low, 0, Token::kERROR); |
| 234 EXPECT(out_of_range_low[0].literal->Equals("invalid code point")); |
| 235 CheckKind(out_of_range_low, 1, Token::kEOS); |
| 236 |
| 237 const Scanner::GrowableTokenStream& out_of_range_high = |
| 238 Scan("\"\\u{FFFFFF}\""); |
| 239 EXPECT_EQ(2, out_of_range_high.length()); |
| 240 CheckKind(out_of_range_high, 0, Token::kERROR); |
| 241 EXPECT(out_of_range_high[0].literal->Equals("invalid code point")); |
| 242 CheckKind(out_of_range_high, 1, Token::kEOS); |
| 243 } |
| 244 |
| 245 |
173 void RawString() { | 246 void RawString() { |
174 // rs = @"\' \\" | 247 // rs = @"\' \\" |
175 const Scanner::GrowableTokenStream& tokens = Scan("rs = @\"\\\' \\\\\""); | 248 const Scanner::GrowableTokenStream& tokens = Scan("rs = @\"\\\' \\\\\""); |
176 | 249 |
177 EXPECT_EQ(4, tokens.length()); | 250 EXPECT_EQ(4, tokens.length()); |
178 CheckIdent(tokens, 0, "rs"); | 251 CheckIdent(tokens, 0, "rs"); |
179 CheckKind(tokens, 1, Token::kASSIGN); | 252 CheckKind(tokens, 1, Token::kASSIGN); |
180 CheckKind(tokens, 2, Token::kSTRING); | 253 CheckKind(tokens, 2, Token::kSTRING); |
181 CheckKind(tokens, 3, Token::kEOS); | 254 CheckKind(tokens, 3, Token::kEOS); |
182 CheckLineNumber(tokens, 2, 1); | 255 CheckLineNumber(tokens, 2, 1); |
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
351 } | 424 } |
352 | 425 |
353 | 426 |
354 TEST_CASE(Scanner_Test) { | 427 TEST_CASE(Scanner_Test) { |
355 ScanLargeText(); | 428 ScanLargeText(); |
356 | 429 |
357 BoringTest(); | 430 BoringTest(); |
358 CommentTest(); | 431 CommentTest(); |
359 GreedIsGood(); | 432 GreedIsGood(); |
360 StringEscapes(); | 433 StringEscapes(); |
| 434 InvalidStringEscapes(); |
361 RawString(); | 435 RawString(); |
362 MultilineString(); | 436 MultilineString(); |
363 EmptyString(); | 437 EmptyString(); |
364 EmptyMultilineString(); | 438 EmptyMultilineString(); |
365 NumberLiteral(); | 439 NumberLiteral(); |
366 InvalidText(); | 440 InvalidText(); |
367 FindLineTest(); | 441 FindLineTest(); |
368 } | 442 } |
369 | 443 |
370 } // namespace dart | 444 } // namespace dart |
OLD | NEW |