Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(38)

Side by Side Diff: runtime/vm/scanner_test.cc

Issue 2481873005: clang-format runtime/vm (Closed)
Patch Set: Merge Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « runtime/vm/scanner.cc ('k') | runtime/vm/scavenger.h » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 #include "platform/assert.h" 5 #include "platform/assert.h"
6 #include "vm/os.h" 6 #include "vm/os.h"
7 #include "vm/scanner.h" 7 #include "vm/scanner.h"
8 #include "vm/token.h" 8 #include "vm/token.h"
9 #include "vm/unit_test.h" 9 #include "vm/unit_test.h"
10 10
11 namespace dart { 11 namespace dart {
12 12
13 13
14 typedef ZoneGrowableArray<Scanner::TokenDescriptor> GrowableTokenStream; 14 typedef ZoneGrowableArray<Scanner::TokenDescriptor> GrowableTokenStream;
15 15
16 16
17 static void LogTokenDesc(Scanner::TokenDescriptor token) { 17 static void LogTokenDesc(Scanner::TokenDescriptor token) {
18 OS::Print("pos %2d:%d-%d token %s ", 18 OS::Print("pos %2d:%d-%d token %s ", token.position.line,
19 token.position.line, token.position.column, 19 token.position.column, token.position.column,
20 token.position.column,
21 Token::Name(token.kind)); 20 Token::Name(token.kind));
22 if (token.literal != NULL) { 21 if (token.literal != NULL) {
23 OS::Print("%s", token.literal->ToCString()); 22 OS::Print("%s", token.literal->ToCString());
24 } 23 }
25 OS::Print("\n"); 24 OS::Print("\n");
26 } 25 }
27 26
28 27
29 static void LogTokenStream(const GrowableTokenStream& token_stream) { 28 static void LogTokenStream(const GrowableTokenStream& token_stream) {
30 int token_index = 0; 29 int token_index = 0;
31 EXPECT_GT(token_stream.length(), 0); 30 EXPECT_GT(token_stream.length(), 0);
32 while (token_index < token_stream.length()) { 31 while (token_index < token_stream.length()) {
33 LogTokenDesc(token_stream[token_index]); 32 LogTokenDesc(token_stream[token_index]);
34 ASSERT(token_stream[token_index].kind != Token::kILLEGAL); 33 ASSERT(token_stream[token_index].kind != Token::kILLEGAL);
35 token_index++; 34 token_index++;
36 } 35 }
37 printf("%d tokens in stream.\n", token_index); 36 printf("%d tokens in stream.\n", token_index);
38 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); 37 EXPECT_EQ(token_stream.Last().kind, Token::kEOS);
39 } 38 }
40 39
41 40
42 static void CheckKind(const GrowableTokenStream &token_stream, 41 static void CheckKind(const GrowableTokenStream& token_stream,
43 int index, 42 int index,
44 Token::Kind kind) { 43 Token::Kind kind) {
45 if (token_stream[index].kind != kind) { 44 if (token_stream[index].kind != kind) {
46 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, 45 OS::PrintErr("Token %d: expected kind %s but got %s\n", index,
47 Token::Name(kind), Token::Name(token_stream[index].kind)); 46 Token::Name(kind), Token::Name(token_stream[index].kind));
48 } 47 }
49 EXPECT_EQ(kind, token_stream[index].kind); 48 EXPECT_EQ(kind, token_stream[index].kind);
50 } 49 }
51 50
52 51
53 static void CheckLiteral(const GrowableTokenStream& token_stream, 52 static void CheckLiteral(const GrowableTokenStream& token_stream,
54 int index, 53 int index,
55 const char* literal) { 54 const char* literal) {
56 if (token_stream[index].literal == NULL) { 55 if (token_stream[index].literal == NULL) {
57 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", 56 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", index,
58 index, literal); 57 literal);
59 } else if (strcmp(literal, token_stream[index].literal->ToCString())) { 58 } else if (strcmp(literal, token_stream[index].literal->ToCString())) {
60 OS::PrintErr("Token %d: expected literal \"%s\" but got \"%s\"\n", 59 OS::PrintErr("Token %d: expected literal \"%s\" but got \"%s\"\n", index,
61 index, literal, token_stream[index].literal->ToCString()); 60 literal, token_stream[index].literal->ToCString());
62 } 61 }
63 } 62 }
64 63
65 64
66 static void CheckIdent(const GrowableTokenStream& token_stream, 65 static void CheckIdent(const GrowableTokenStream& token_stream,
67 int index, 66 int index,
68 const char* literal) { 67 const char* literal) {
69 CheckKind(token_stream, index, Token::kIDENT); 68 CheckKind(token_stream, index, Token::kIDENT);
70 CheckLiteral(token_stream, index, literal); 69 CheckLiteral(token_stream, index, literal);
71 } 70 }
72 71
73 72
74 static void CheckInteger(const GrowableTokenStream& token_stream, 73 static void CheckInteger(const GrowableTokenStream& token_stream,
75 int index, 74 int index,
76 const char* literal) { 75 const char* literal) {
77 CheckKind(token_stream, index, Token::kINTEGER); 76 CheckKind(token_stream, index, Token::kINTEGER);
78 CheckLiteral(token_stream, index, literal); 77 CheckLiteral(token_stream, index, literal);
79 } 78 }
80 79
81 80
82 static void CheckLineNumber(const GrowableTokenStream& token_stream, 81 static void CheckLineNumber(const GrowableTokenStream& token_stream,
83 int index, 82 int index,
84 int line_number) { 83 int line_number) {
85 if (token_stream[index].position.line != line_number) { 84 if (token_stream[index].position.line != line_number) {
86 OS::PrintErr("Token %d: expected line number %d but got %d\n", 85 OS::PrintErr("Token %d: expected line number %d but got %d\n", index,
87 index, line_number, token_stream[index].position.line); 86 line_number, token_stream[index].position.line);
88 } 87 }
89 } 88 }
90 89
91 90
92 static void CheckNumTokens(const GrowableTokenStream& token_stream, 91 static void CheckNumTokens(const GrowableTokenStream& token_stream, int index) {
93 int index) {
94 if (token_stream.length() != index) { 92 if (token_stream.length() != index) {
95 OS::PrintErr("Expected %d tokens but got only %" Pd ".\n", 93 OS::PrintErr("Expected %d tokens but got only %" Pd ".\n", index,
96 index, token_stream.length()); 94 token_stream.length());
97 } 95 }
98 } 96 }
99 97
100 98
101 class Collector : public Scanner::TokenCollector { 99 class Collector : public Scanner::TokenCollector {
102 public: 100 public:
103 explicit Collector(GrowableTokenStream* ts) : ts_(ts) { } 101 explicit Collector(GrowableTokenStream* ts) : ts_(ts) {}
104 virtual ~Collector() { } 102 virtual ~Collector() {}
105 103
106 virtual void AddToken(const Scanner::TokenDescriptor& token) { 104 virtual void AddToken(const Scanner::TokenDescriptor& token) {
107 ts_->Add(token); 105 ts_->Add(token);
108 } 106 }
107
109 private: 108 private:
110 GrowableTokenStream* ts_; 109 GrowableTokenStream* ts_;
111 }; 110 };
112 111
113 112
114 static const GrowableTokenStream& Scan(const char* source) { 113 static const GrowableTokenStream& Scan(const char* source) {
115 OS::Print("\nScanning: <%s>\n", source); 114 OS::Print("\nScanning: <%s>\n", source);
116 115
117 Scanner scanner(String::Handle(String::New(source)), 116 Scanner scanner(String::Handle(String::New(source)),
118 String::Handle(String::New(""))); 117 String::Handle(String::New("")));
(...skipping 12 matching lines...) Expand all
131 CheckNumTokens(tokens, 6); 130 CheckNumTokens(tokens, 6);
132 CheckIdent(tokens, 0, "x"); 131 CheckIdent(tokens, 0, "x");
133 CheckKind(tokens, 1, Token::kASSIGN); 132 CheckKind(tokens, 1, Token::kASSIGN);
134 CheckIdent(tokens, 2, "iffy"); 133 CheckIdent(tokens, 2, "iffy");
135 CheckKind(tokens, 3, Token::kINCR); 134 CheckKind(tokens, 3, Token::kINCR);
136 CheckKind(tokens, 4, Token::kSEMICOLON); 135 CheckKind(tokens, 4, Token::kSEMICOLON);
137 } 136 }
138 137
139 138
140 static void CommentTest() { 139 static void CommentTest() {
141 const GrowableTokenStream& tokens = 140 const GrowableTokenStream& tokens = Scan(
142 Scan("Foo( /*block \n" 141 "Foo( /*block \n"
143 "comment*/ 0xff) // line comment;"); 142 "comment*/ 0xff) // line comment;");
144 143
145 CheckNumTokens(tokens, 6); 144 CheckNumTokens(tokens, 6);
146 CheckIdent(tokens, 0, "Foo"); 145 CheckIdent(tokens, 0, "Foo");
147 CheckLineNumber(tokens, 0, 1); 146 CheckLineNumber(tokens, 0, 1);
148 CheckKind(tokens, 1, Token::kLPAREN); 147 CheckKind(tokens, 1, Token::kLPAREN);
149 CheckKind(tokens, 2, Token::kNEWLINE); 148 CheckKind(tokens, 2, Token::kNEWLINE);
150 CheckInteger(tokens, 3, "0xff"); 149 CheckInteger(tokens, 3, "0xff");
151 CheckKind(tokens, 4, Token::kRPAREN); 150 CheckKind(tokens, 4, Token::kRPAREN);
152 CheckLineNumber(tokens, 4, 2); 151 CheckLineNumber(tokens, 4, 2);
153 } 152 }
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
185 EXPECT_EQ(' ', litchars[1]); 184 EXPECT_EQ(' ', litchars[1]);
186 EXPECT_EQ('\\', litchars[2]); 185 EXPECT_EQ('\\', litchars[2]);
187 EXPECT_EQ('\n', litchars[4]); 186 EXPECT_EQ('\n', litchars[4]);
188 EXPECT_EQ('\r', litchars[5]); 187 EXPECT_EQ('\r', litchars[5]);
189 EXPECT_EQ('\t', litchars[6]); 188 EXPECT_EQ('\t', litchars[6]);
190 EXPECT_EQ('\'', litchars[8]); 189 EXPECT_EQ('\'', litchars[8]);
191 } 190 }
192 191
193 192
194 static void InvalidStringEscapes() { 193 static void InvalidStringEscapes() {
195 const GrowableTokenStream& out_of_range_low = 194 const GrowableTokenStream& out_of_range_low = Scan("\"\\u{110000}\"");
196 Scan("\"\\u{110000}\"");
197 EXPECT_EQ(2, out_of_range_low.length()); 195 EXPECT_EQ(2, out_of_range_low.length());
198 CheckKind(out_of_range_low, 0, Token::kERROR); 196 CheckKind(out_of_range_low, 0, Token::kERROR);
199 EXPECT(out_of_range_low[0].literal->Equals("invalid code point")); 197 EXPECT(out_of_range_low[0].literal->Equals("invalid code point"));
200 CheckKind(out_of_range_low, 1, Token::kEOS); 198 CheckKind(out_of_range_low, 1, Token::kEOS);
201 199
202 const GrowableTokenStream& out_of_range_high = 200 const GrowableTokenStream& out_of_range_high = Scan("\"\\u{FFFFFF}\"");
203 Scan("\"\\u{FFFFFF}\"");
204 EXPECT_EQ(2, out_of_range_high.length()); 201 EXPECT_EQ(2, out_of_range_high.length());
205 CheckKind(out_of_range_high, 0, Token::kERROR); 202 CheckKind(out_of_range_high, 0, Token::kERROR);
206 EXPECT(out_of_range_high[0].literal->Equals("invalid code point")); 203 EXPECT(out_of_range_high[0].literal->Equals("invalid code point"));
207 CheckKind(out_of_range_high, 1, Token::kEOS); 204 CheckKind(out_of_range_high, 1, Token::kEOS);
208 } 205 }
209 206
210 207
211 static void RawString() { 208 static void RawString() {
212 // rs = @"\' \\" 209 // rs = @"\' \\"
213 const GrowableTokenStream& tokens = Scan("rs = r\"\\\' \\\\\""); 210 const GrowableTokenStream& tokens = Scan("rs = r\"\\\' \\\\\"");
214 211
215 EXPECT_EQ(4, tokens.length()); 212 EXPECT_EQ(4, tokens.length());
216 CheckIdent(tokens, 0, "rs"); 213 CheckIdent(tokens, 0, "rs");
217 CheckKind(tokens, 1, Token::kASSIGN); 214 CheckKind(tokens, 1, Token::kASSIGN);
218 CheckKind(tokens, 2, Token::kSTRING); 215 CheckKind(tokens, 2, Token::kSTRING);
219 CheckKind(tokens, 3, Token::kEOS); 216 CheckKind(tokens, 3, Token::kEOS);
220 CheckLineNumber(tokens, 2, 1); 217 CheckLineNumber(tokens, 2, 1);
221 const char* litchars = (tokens)[2].literal->ToCString(); 218 const char* litchars = (tokens)[2].literal->ToCString();
222 EXPECT_EQ(5, (tokens)[2].literal->Length()); 219 EXPECT_EQ(5, (tokens)[2].literal->Length());
223 220
224 EXPECT_EQ('\\', litchars[0]); 221 EXPECT_EQ('\\', litchars[0]);
225 EXPECT_EQ('\'', litchars[1]); 222 EXPECT_EQ('\'', litchars[1]);
226 EXPECT_EQ(' ', litchars[2]); 223 EXPECT_EQ(' ', litchars[2]);
227 EXPECT_EQ('\\', litchars[3]); 224 EXPECT_EQ('\\', litchars[3]);
228 EXPECT_EQ('\\', litchars[4]); 225 EXPECT_EQ('\\', litchars[4]);
229 } 226 }
230 227
231 228
232 static void MultilineString() { 229 static void MultilineString() {
233 // |mls = ''' 230 // |mls = '''
234 // |1' x 231 // |1' x
235 // |2'''; 232 // |2''';
236 const GrowableTokenStream& tokens = Scan("mls = '''\n1' x\n2''';"); 233 const GrowableTokenStream& tokens = Scan("mls = '''\n1' x\n2''';");
237 234
238 EXPECT_EQ(7, tokens.length()); 235 EXPECT_EQ(7, tokens.length());
239 CheckIdent(tokens, 0, "mls"); 236 CheckIdent(tokens, 0, "mls");
240 CheckKind(tokens, 1, Token::kASSIGN); 237 CheckKind(tokens, 1, Token::kASSIGN);
241 CheckKind(tokens, 2, Token::kSTRING); 238 CheckKind(tokens, 2, Token::kSTRING);
242 CheckKind(tokens, 3, Token::kNEWLINE); 239 CheckKind(tokens, 3, Token::kNEWLINE);
243 CheckKind(tokens, 4, Token::kNEWLINE); 240 CheckKind(tokens, 4, Token::kNEWLINE);
244 CheckKind(tokens, 5, Token::kSEMICOLON); 241 CheckKind(tokens, 5, Token::kSEMICOLON);
245 CheckKind(tokens, 6, Token::kEOS); 242 CheckKind(tokens, 6, Token::kEOS);
246 CheckLineNumber(tokens, 0, 1); 243 CheckLineNumber(tokens, 0, 1);
247 CheckLineNumber(tokens, 5, 3); // Semicolon is on line 3. 244 CheckLineNumber(tokens, 5, 3); // Semicolon is on line 3.
248 const char* litchars = (tokens)[2].literal->ToCString(); 245 const char* litchars = (tokens)[2].literal->ToCString();
249 EXPECT_EQ(6, (tokens)[2].literal->Length()); 246 EXPECT_EQ(6, (tokens)[2].literal->Length());
250 247
251 EXPECT_EQ('1', litchars[0]); // First newline is dropped. 248 EXPECT_EQ('1', litchars[0]); // First newline is dropped.
252 EXPECT_EQ('\'', litchars[1]); 249 EXPECT_EQ('\'', litchars[1]);
253 EXPECT_EQ(' ', litchars[2]); 250 EXPECT_EQ(' ', litchars[2]);
254 EXPECT_EQ('x', litchars[3]); 251 EXPECT_EQ('x', litchars[3]);
255 EXPECT_EQ('\n', litchars[4]); 252 EXPECT_EQ('\n', litchars[4]);
256 EXPECT_EQ('2', litchars[5]); 253 EXPECT_EQ('2', litchars[5]);
257 } 254 }
258 255
259 256
260 static void EmptyString() { 257 static void EmptyString() {
261 // es = ""; 258 // es = "";
262 const GrowableTokenStream& tokens = Scan("es = \"\";"); 259 const GrowableTokenStream& tokens = Scan("es = \"\";");
263 260
264 EXPECT_EQ(5, tokens.length()); 261 EXPECT_EQ(5, tokens.length());
265 CheckIdent(tokens, 0, "es"); 262 CheckIdent(tokens, 0, "es");
266 CheckKind(tokens, 1, Token::kASSIGN); 263 CheckKind(tokens, 1, Token::kASSIGN);
(...skipping 11 matching lines...) Expand all
278 CheckIdent(tokens, 0, "es"); 275 CheckIdent(tokens, 0, "es");
279 CheckKind(tokens, 1, Token::kASSIGN); 276 CheckKind(tokens, 1, Token::kASSIGN);
280 CheckKind(tokens, 2, Token::kSTRING); 277 CheckKind(tokens, 2, Token::kSTRING);
281 CheckKind(tokens, 3, Token::kSEMICOLON); 278 CheckKind(tokens, 3, Token::kSEMICOLON);
282 CheckKind(tokens, 4, Token::kEOS); 279 CheckKind(tokens, 4, Token::kEOS);
283 EXPECT_EQ(0, (tokens)[2].literal->Length()); 280 EXPECT_EQ(0, (tokens)[2].literal->Length());
284 } 281 }
285 282
286 283
287 static void NumberLiteral() { 284 static void NumberLiteral() {
288 const GrowableTokenStream& tokens = 285 const GrowableTokenStream& tokens = Scan("5 0x5d 0.3 0.33 1E+12 .42 +5");
289 Scan("5 0x5d 0.3 0.33 1E+12 .42 +5");
290 286
291 CheckKind(tokens, 0, Token::kINTEGER); 287 CheckKind(tokens, 0, Token::kINTEGER);
292 CheckKind(tokens, 1, Token::kINTEGER); 288 CheckKind(tokens, 1, Token::kINTEGER);
293 CheckKind(tokens, 2, Token::kDOUBLE); 289 CheckKind(tokens, 2, Token::kDOUBLE);
294 CheckKind(tokens, 3, Token::kDOUBLE); 290 CheckKind(tokens, 3, Token::kDOUBLE);
295 CheckKind(tokens, 4, Token::kDOUBLE); 291 CheckKind(tokens, 4, Token::kDOUBLE);
296 CheckKind(tokens, 5, Token::kDOUBLE); 292 CheckKind(tokens, 5, Token::kDOUBLE);
297 CheckKind(tokens, 6, Token::kADD); 293 CheckKind(tokens, 6, Token::kADD);
298 CheckKind(tokens, 7, Token::kINTEGER); 294 CheckKind(tokens, 7, Token::kINTEGER);
299 CheckKind(tokens, 8, Token::kEOS); 295 CheckKind(tokens, 8, Token::kEOS);
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after
358 " }" 354 " }"
359 "}" 355 "}"
360 "" 356 ""
361 "" 357 ""
362 "j!==!iffy // means j !== !iffy"; 358 "j!==!iffy // means j !== !iffy";
363 Scan(dart_source); 359 Scan(dart_source);
364 } 360 }
365 361
366 362
367 void InvalidText() { 363 void InvalidText() {
368 const GrowableTokenStream& tokens = 364 const GrowableTokenStream& tokens = Scan("\\");
369 Scan("\\");
370 365
371 EXPECT_EQ(2, tokens.length()); 366 EXPECT_EQ(2, tokens.length());
372 CheckKind(tokens, 0, Token::kERROR); 367 CheckKind(tokens, 0, Token::kERROR);
373 CheckKind(tokens, 1, Token::kEOS); 368 CheckKind(tokens, 1, Token::kEOS);
374 } 369 }
375 370
376 371
377 void NewlinesTest() { 372 void NewlinesTest() {
378 const char* source = 373 const char* source =
379 "var es = /* a\n" 374 "var es = /* a\n"
(...skipping 13 matching lines...) Expand all
393 CheckKind(tokens, 4, Token::kNEWLINE); 388 CheckKind(tokens, 4, Token::kNEWLINE);
394 CheckKind(tokens, 5, Token::kSTRING); 389 CheckKind(tokens, 5, Token::kSTRING);
395 CheckKind(tokens, 6, Token::kNEWLINE); 390 CheckKind(tokens, 6, Token::kNEWLINE);
396 CheckKind(tokens, 7, Token::kNEWLINE); 391 CheckKind(tokens, 7, Token::kNEWLINE);
397 CheckKind(tokens, 8, Token::kNEWLINE); 392 CheckKind(tokens, 8, Token::kNEWLINE);
398 CheckKind(tokens, 9, Token::kSEMICOLON); 393 CheckKind(tokens, 9, Token::kSEMICOLON);
399 CheckKind(tokens, 10, Token::kEOS); 394 CheckKind(tokens, 10, Token::kEOS);
400 395
401 EXPECT_EQ(4, (tokens)[5].literal->Length()); 396 EXPECT_EQ(4, (tokens)[5].literal->Length());
402 const char* litchars = (tokens)[5].literal->ToCString(); 397 const char* litchars = (tokens)[5].literal->ToCString();
403 EXPECT_EQ('c', litchars[0]); // First newline is dropped. 398 EXPECT_EQ('c', litchars[0]); // First newline is dropped.
404 EXPECT_EQ('\n', litchars[1]); 399 EXPECT_EQ('\n', litchars[1]);
405 EXPECT_EQ('d', litchars[2]); 400 EXPECT_EQ('d', litchars[2]);
406 EXPECT_EQ('\n', litchars[3]); 401 EXPECT_EQ('\n', litchars[3]);
407 } 402 }
408 403
409 404
410 TEST_CASE(Scanner_Test) { 405 TEST_CASE(Scanner_Test) {
411 ScanLargeText(); 406 ScanLargeText();
412 407
413 BoringTest(); 408 BoringTest();
414 CommentTest(); 409 CommentTest();
415 GreedIsGood(); 410 GreedIsGood();
416 StringEscapes(); 411 StringEscapes();
417 InvalidStringEscapes(); 412 InvalidStringEscapes();
418 RawString(); 413 RawString();
419 MultilineString(); 414 MultilineString();
420 EmptyString(); 415 EmptyString();
421 EmptyMultilineString(); 416 EmptyMultilineString();
422 NumberLiteral(); 417 NumberLiteral();
423 InvalidText(); 418 InvalidText();
424 NewlinesTest(); 419 NewlinesTest();
425 } 420 }
426 421
427 } // namespace dart 422 } // namespace dart
OLDNEW
« no previous file with comments | « runtime/vm/scanner.cc ('k') | runtime/vm/scavenger.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698