| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "platform/assert.h" | 5 #include "platform/assert.h" |
| 6 #include "vm/os.h" | 6 #include "vm/os.h" |
| 7 #include "vm/scanner.h" | 7 #include "vm/scanner.h" |
| 8 #include "vm/token.h" | 8 #include "vm/token.h" |
| 9 #include "vm/unit_test.h" | 9 #include "vm/unit_test.h" |
| 10 | 10 |
| (...skipping 22 matching lines...) Expand all Loading... |
| 33 printf("%d tokens in stream.\n", token_index); | 33 printf("%d tokens in stream.\n", token_index); |
| 34 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); | 34 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); |
| 35 } | 35 } |
| 36 | 36 |
| 37 | 37 |
| 38 void CheckKind(const Scanner::GrowableTokenStream &token_stream, | 38 void CheckKind(const Scanner::GrowableTokenStream &token_stream, |
| 39 int index, | 39 int index, |
| 40 Token::Kind kind) { | 40 Token::Kind kind) { |
| 41 if (token_stream[index].kind != kind) { | 41 if (token_stream[index].kind != kind) { |
| 42 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, | 42 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, |
| 43 Token::Name(token_stream[index].kind), Token::Name(kind)); | 43 Token::Name(kind), Token::Name(token_stream[index].kind)); |
| 44 } | 44 } |
| 45 EXPECT_EQ(kind, token_stream[index].kind); | 45 EXPECT_EQ(kind, token_stream[index].kind); |
| 46 } | 46 } |
| 47 | 47 |
| 48 | 48 |
| 49 void CheckLiteral(const Scanner::GrowableTokenStream& token_stream, | 49 void CheckLiteral(const Scanner::GrowableTokenStream& token_stream, |
| 50 int index, | 50 int index, |
| 51 const char* literal) { | 51 const char* literal) { |
| 52 if (token_stream[index].literal == NULL) { | 52 if (token_stream[index].literal == NULL) { |
| 53 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", | 53 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", |
| (...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 163 EXPECT_EQ('"', litchars[0]); | 163 EXPECT_EQ('"', litchars[0]); |
| 164 EXPECT_EQ(' ', litchars[1]); | 164 EXPECT_EQ(' ', litchars[1]); |
| 165 EXPECT_EQ('\\', litchars[2]); | 165 EXPECT_EQ('\\', litchars[2]); |
| 166 EXPECT_EQ('\n', litchars[4]); | 166 EXPECT_EQ('\n', litchars[4]); |
| 167 EXPECT_EQ('\r', litchars[5]); | 167 EXPECT_EQ('\r', litchars[5]); |
| 168 EXPECT_EQ('\t', litchars[6]); | 168 EXPECT_EQ('\t', litchars[6]); |
| 169 EXPECT_EQ('\'', litchars[8]); | 169 EXPECT_EQ('\'', litchars[8]); |
| 170 } | 170 } |
| 171 | 171 |
| 172 | 172 |
| 173 void InvalidStringEscapes() { |
| 174 const Scanner::GrowableTokenStream& high_start_4 = |
| 175 Scan("\"\\uD800\""); |
| 176 EXPECT_EQ(2, high_start_4.length()); |
| 177 CheckKind(high_start_4, 0, Token::kERROR); |
| 178 EXPECT(high_start_4[0].literal->Equals("invalid code point")); |
| 179 CheckKind(high_start_4, 1, Token::kEOS); |
| 180 |
| 181 const Scanner::GrowableTokenStream& high_start_seq = |
| 182 Scan("\"\\u{D800}\""); |
| 183 EXPECT_EQ(2, high_start_seq.length()); |
| 184 CheckKind(high_start_seq, 0, Token::kERROR); |
| 185 EXPECT(high_start_seq[0].literal->Equals("invalid code point")); |
| 186 CheckKind(high_start_seq, 1, Token::kEOS); |
| 187 |
| 188 const Scanner::GrowableTokenStream& high_end_4 = |
| 189 Scan("\"\\uDBFF\""); |
| 190 EXPECT_EQ(2, high_end_4.length()); |
| 191 CheckKind(high_end_4, 0, Token::kERROR); |
| 192 EXPECT(high_end_4[0].literal->Equals("invalid code point")); |
| 193 CheckKind(high_end_4, 1, Token::kEOS); |
| 194 |
| 195 const Scanner::GrowableTokenStream& high_end_seq = |
| 196 Scan("\"\\u{DBFF}\""); |
| 197 EXPECT_EQ(2, high_end_seq.length()); |
| 198 CheckKind(high_end_seq, 0, Token::kERROR); |
| 199 EXPECT(high_end_seq[0].literal->Equals("invalid code point")); |
| 200 CheckKind(high_end_seq, 1, Token::kEOS); |
| 201 |
| 202 const Scanner::GrowableTokenStream& low_start_4 = |
| 203 Scan("\"\\uDC00\""); |
| 204 EXPECT_EQ(2, low_start_4.length()); |
| 205 CheckKind(low_start_4, 0, Token::kERROR); |
| 206 EXPECT(low_start_4[0].literal->Equals("invalid code point")); |
| 207 CheckKind(low_start_4, 1, Token::kEOS); |
| 208 |
| 209 const Scanner::GrowableTokenStream& low_start_seq = |
| 210 Scan("\"\\u{DC00}\""); |
| 211 EXPECT_EQ(2, low_start_seq.length()); |
| 212 CheckKind(low_start_seq, 0, Token::kERROR); |
| 213 EXPECT(low_start_seq[0].literal->Equals("invalid code point")); |
| 214 CheckKind(low_start_seq, 1, Token::kEOS); |
| 215 |
| 216 const Scanner::GrowableTokenStream& low_end_4 = |
| 217 Scan("\"\\uDFFF\""); |
| 218 EXPECT_EQ(2, low_end_4.length()); |
| 219 CheckKind(low_end_4, 0, Token::kERROR); |
| 220 EXPECT(low_end_4[0].literal->Equals("invalid code point")); |
| 221 CheckKind(low_end_4, 1, Token::kEOS); |
| 222 |
| 223 const Scanner::GrowableTokenStream& low_end_seq = |
| 224 Scan("\"\\u{DFFF}\""); |
| 225 EXPECT_EQ(2, low_end_seq.length()); |
| 226 CheckKind(low_end_seq, 0, Token::kERROR); |
| 227 EXPECT(low_end_seq[0].literal->Equals("invalid code point")); |
| 228 CheckKind(low_end_seq, 1, Token::kEOS); |
| 229 |
| 230 const Scanner::GrowableTokenStream& out_of_range_low = |
| 231 Scan("\"\\u{110000}\""); |
| 232 EXPECT_EQ(2, out_of_range_low.length()); |
| 233 CheckKind(out_of_range_low, 0, Token::kERROR); |
| 234 EXPECT(out_of_range_low[0].literal->Equals("invalid code point")); |
| 235 CheckKind(out_of_range_low, 1, Token::kEOS); |
| 236 |
| 237 const Scanner::GrowableTokenStream& out_of_range_high = |
| 238 Scan("\"\\u{FFFFFF}\""); |
| 239 EXPECT_EQ(2, out_of_range_high.length()); |
| 240 CheckKind(out_of_range_high, 0, Token::kERROR); |
| 241 EXPECT(out_of_range_high[0].literal->Equals("invalid code point")); |
| 242 CheckKind(out_of_range_high, 1, Token::kEOS); |
| 243 } |
| 244 |
| 245 |
| 173 void RawString() { | 246 void RawString() { |
| 174 // rs = @"\' \\" | 247 // rs = @"\' \\" |
| 175 const Scanner::GrowableTokenStream& tokens = Scan("rs = @\"\\\' \\\\\""); | 248 const Scanner::GrowableTokenStream& tokens = Scan("rs = @\"\\\' \\\\\""); |
| 176 | 249 |
| 177 EXPECT_EQ(4, tokens.length()); | 250 EXPECT_EQ(4, tokens.length()); |
| 178 CheckIdent(tokens, 0, "rs"); | 251 CheckIdent(tokens, 0, "rs"); |
| 179 CheckKind(tokens, 1, Token::kASSIGN); | 252 CheckKind(tokens, 1, Token::kASSIGN); |
| 180 CheckKind(tokens, 2, Token::kSTRING); | 253 CheckKind(tokens, 2, Token::kSTRING); |
| 181 CheckKind(tokens, 3, Token::kEOS); | 254 CheckKind(tokens, 3, Token::kEOS); |
| 182 CheckLineNumber(tokens, 2, 1); | 255 CheckLineNumber(tokens, 2, 1); |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 351 } | 424 } |
| 352 | 425 |
| 353 | 426 |
| 354 TEST_CASE(Scanner_Test) { | 427 TEST_CASE(Scanner_Test) { |
| 355 ScanLargeText(); | 428 ScanLargeText(); |
| 356 | 429 |
| 357 BoringTest(); | 430 BoringTest(); |
| 358 CommentTest(); | 431 CommentTest(); |
| 359 GreedIsGood(); | 432 GreedIsGood(); |
| 360 StringEscapes(); | 433 StringEscapes(); |
| 434 InvalidStringEscapes(); |
| 361 RawString(); | 435 RawString(); |
| 362 MultilineString(); | 436 MultilineString(); |
| 363 EmptyString(); | 437 EmptyString(); |
| 364 EmptyMultilineString(); | 438 EmptyMultilineString(); |
| 365 NumberLiteral(); | 439 NumberLiteral(); |
| 366 InvalidText(); | 440 InvalidText(); |
| 367 FindLineTest(); | 441 FindLineTest(); |
| 368 } | 442 } |
| 369 | 443 |
| 370 } // namespace dart | 444 } // namespace dart |
| OLD | NEW |