OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
| 5 #include "vm/scanner.h" |
5 #include "platform/assert.h" | 6 #include "platform/assert.h" |
6 #include "vm/os.h" | 7 #include "vm/os.h" |
7 #include "vm/scanner.h" | |
8 #include "vm/token.h" | 8 #include "vm/token.h" |
9 #include "vm/unit_test.h" | 9 #include "vm/unit_test.h" |
10 | 10 |
11 namespace dart { | 11 namespace dart { |
12 | 12 |
13 | |
14 typedef ZoneGrowableArray<Scanner::TokenDescriptor> GrowableTokenStream; | 13 typedef ZoneGrowableArray<Scanner::TokenDescriptor> GrowableTokenStream; |
15 | 14 |
16 | |
17 static void LogTokenDesc(Scanner::TokenDescriptor token) { | 15 static void LogTokenDesc(Scanner::TokenDescriptor token) { |
18 OS::Print("pos %2d:%d-%d token %s ", token.position.line, | 16 OS::Print("pos %2d:%d-%d token %s ", token.position.line, |
19 token.position.column, token.position.column, | 17 token.position.column, token.position.column, |
20 Token::Name(token.kind)); | 18 Token::Name(token.kind)); |
21 if (token.literal != NULL) { | 19 if (token.literal != NULL) { |
22 OS::Print("%s", token.literal->ToCString()); | 20 OS::Print("%s", token.literal->ToCString()); |
23 } | 21 } |
24 OS::Print("\n"); | 22 OS::Print("\n"); |
25 } | 23 } |
26 | 24 |
27 | |
28 static void LogTokenStream(const GrowableTokenStream& token_stream) { | 25 static void LogTokenStream(const GrowableTokenStream& token_stream) { |
29 int token_index = 0; | 26 int token_index = 0; |
30 EXPECT_GT(token_stream.length(), 0); | 27 EXPECT_GT(token_stream.length(), 0); |
31 while (token_index < token_stream.length()) { | 28 while (token_index < token_stream.length()) { |
32 LogTokenDesc(token_stream[token_index]); | 29 LogTokenDesc(token_stream[token_index]); |
33 ASSERT(token_stream[token_index].kind != Token::kILLEGAL); | 30 ASSERT(token_stream[token_index].kind != Token::kILLEGAL); |
34 token_index++; | 31 token_index++; |
35 } | 32 } |
36 printf("%d tokens in stream.\n", token_index); | 33 printf("%d tokens in stream.\n", token_index); |
37 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); | 34 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); |
38 } | 35 } |
39 | 36 |
40 | |
41 static void CheckKind(const GrowableTokenStream& token_stream, | 37 static void CheckKind(const GrowableTokenStream& token_stream, |
42 int index, | 38 int index, |
43 Token::Kind kind) { | 39 Token::Kind kind) { |
44 if (token_stream[index].kind != kind) { | 40 if (token_stream[index].kind != kind) { |
45 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, | 41 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, |
46 Token::Name(kind), Token::Name(token_stream[index].kind)); | 42 Token::Name(kind), Token::Name(token_stream[index].kind)); |
47 } | 43 } |
48 EXPECT_EQ(kind, token_stream[index].kind); | 44 EXPECT_EQ(kind, token_stream[index].kind); |
49 } | 45 } |
50 | 46 |
51 | |
52 static void CheckLiteral(const GrowableTokenStream& token_stream, | 47 static void CheckLiteral(const GrowableTokenStream& token_stream, |
53 int index, | 48 int index, |
54 const char* literal) { | 49 const char* literal) { |
55 if (token_stream[index].literal == NULL) { | 50 if (token_stream[index].literal == NULL) { |
56 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", index, | 51 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", index, |
57 literal); | 52 literal); |
58 } else if (strcmp(literal, token_stream[index].literal->ToCString())) { | 53 } else if (strcmp(literal, token_stream[index].literal->ToCString())) { |
59 OS::PrintErr("Token %d: expected literal \"%s\" but got \"%s\"\n", index, | 54 OS::PrintErr("Token %d: expected literal \"%s\" but got \"%s\"\n", index, |
60 literal, token_stream[index].literal->ToCString()); | 55 literal, token_stream[index].literal->ToCString()); |
61 } | 56 } |
62 } | 57 } |
63 | 58 |
64 | |
65 static void CheckIdent(const GrowableTokenStream& token_stream, | 59 static void CheckIdent(const GrowableTokenStream& token_stream, |
66 int index, | 60 int index, |
67 const char* literal) { | 61 const char* literal) { |
68 CheckKind(token_stream, index, Token::kIDENT); | 62 CheckKind(token_stream, index, Token::kIDENT); |
69 CheckLiteral(token_stream, index, literal); | 63 CheckLiteral(token_stream, index, literal); |
70 } | 64 } |
71 | 65 |
72 | |
73 static void CheckInteger(const GrowableTokenStream& token_stream, | 66 static void CheckInteger(const GrowableTokenStream& token_stream, |
74 int index, | 67 int index, |
75 const char* literal) { | 68 const char* literal) { |
76 CheckKind(token_stream, index, Token::kINTEGER); | 69 CheckKind(token_stream, index, Token::kINTEGER); |
77 CheckLiteral(token_stream, index, literal); | 70 CheckLiteral(token_stream, index, literal); |
78 } | 71 } |
79 | 72 |
80 | |
81 static void CheckLineNumber(const GrowableTokenStream& token_stream, | 73 static void CheckLineNumber(const GrowableTokenStream& token_stream, |
82 int index, | 74 int index, |
83 int line_number) { | 75 int line_number) { |
84 if (token_stream[index].position.line != line_number) { | 76 if (token_stream[index].position.line != line_number) { |
85 OS::PrintErr("Token %d: expected line number %d but got %d\n", index, | 77 OS::PrintErr("Token %d: expected line number %d but got %d\n", index, |
86 line_number, token_stream[index].position.line); | 78 line_number, token_stream[index].position.line); |
87 } | 79 } |
88 } | 80 } |
89 | 81 |
90 | |
91 static void CheckNumTokens(const GrowableTokenStream& token_stream, int index) { | 82 static void CheckNumTokens(const GrowableTokenStream& token_stream, int index) { |
92 if (token_stream.length() != index) { | 83 if (token_stream.length() != index) { |
93 OS::PrintErr("Expected %d tokens but got only %" Pd ".\n", index, | 84 OS::PrintErr("Expected %d tokens but got only %" Pd ".\n", index, |
94 token_stream.length()); | 85 token_stream.length()); |
95 } | 86 } |
96 } | 87 } |
97 | 88 |
98 | |
99 class Collector : public Scanner::TokenCollector { | 89 class Collector : public Scanner::TokenCollector { |
100 public: | 90 public: |
101 explicit Collector(GrowableTokenStream* ts) : ts_(ts) {} | 91 explicit Collector(GrowableTokenStream* ts) : ts_(ts) {} |
102 virtual ~Collector() {} | 92 virtual ~Collector() {} |
103 | 93 |
104 virtual void AddToken(const Scanner::TokenDescriptor& token) { | 94 virtual void AddToken(const Scanner::TokenDescriptor& token) { |
105 ts_->Add(token); | 95 ts_->Add(token); |
106 } | 96 } |
107 | 97 |
108 private: | 98 private: |
109 GrowableTokenStream* ts_; | 99 GrowableTokenStream* ts_; |
110 }; | 100 }; |
111 | 101 |
112 | |
113 static const GrowableTokenStream& Scan(const char* source) { | 102 static const GrowableTokenStream& Scan(const char* source) { |
114 OS::Print("\nScanning: <%s>\n", source); | 103 OS::Print("\nScanning: <%s>\n", source); |
115 | 104 |
116 Scanner scanner(String::Handle(String::New(source)), | 105 Scanner scanner(String::Handle(String::New(source)), |
117 String::Handle(String::New(""))); | 106 String::Handle(String::New(""))); |
118 GrowableTokenStream* tokens = new GrowableTokenStream(128); | 107 GrowableTokenStream* tokens = new GrowableTokenStream(128); |
119 Collector collector(tokens); | 108 Collector collector(tokens); |
120 | 109 |
121 scanner.ScanAll(&collector); | 110 scanner.ScanAll(&collector); |
122 LogTokenStream(*tokens); | 111 LogTokenStream(*tokens); |
123 return *tokens; | 112 return *tokens; |
124 } | 113 } |
125 | 114 |
126 | |
127 static void BoringTest() { | 115 static void BoringTest() { |
128 const GrowableTokenStream& tokens = Scan("x = iffy++;"); | 116 const GrowableTokenStream& tokens = Scan("x = iffy++;"); |
129 | 117 |
130 CheckNumTokens(tokens, 6); | 118 CheckNumTokens(tokens, 6); |
131 CheckIdent(tokens, 0, "x"); | 119 CheckIdent(tokens, 0, "x"); |
132 CheckKind(tokens, 1, Token::kASSIGN); | 120 CheckKind(tokens, 1, Token::kASSIGN); |
133 CheckIdent(tokens, 2, "iffy"); | 121 CheckIdent(tokens, 2, "iffy"); |
134 CheckKind(tokens, 3, Token::kINCR); | 122 CheckKind(tokens, 3, Token::kINCR); |
135 CheckKind(tokens, 4, Token::kSEMICOLON); | 123 CheckKind(tokens, 4, Token::kSEMICOLON); |
136 } | 124 } |
137 | 125 |
138 | |
139 static void CommentTest() { | 126 static void CommentTest() { |
140 const GrowableTokenStream& tokens = Scan( | 127 const GrowableTokenStream& tokens = Scan( |
141 "Foo( /*block \n" | 128 "Foo( /*block \n" |
142 "comment*/ 0xff) // line comment;"); | 129 "comment*/ 0xff) // line comment;"); |
143 | 130 |
144 CheckNumTokens(tokens, 6); | 131 CheckNumTokens(tokens, 6); |
145 CheckIdent(tokens, 0, "Foo"); | 132 CheckIdent(tokens, 0, "Foo"); |
146 CheckLineNumber(tokens, 0, 1); | 133 CheckLineNumber(tokens, 0, 1); |
147 CheckKind(tokens, 1, Token::kLPAREN); | 134 CheckKind(tokens, 1, Token::kLPAREN); |
148 CheckKind(tokens, 2, Token::kNEWLINE); | 135 CheckKind(tokens, 2, Token::kNEWLINE); |
149 CheckInteger(tokens, 3, "0xff"); | 136 CheckInteger(tokens, 3, "0xff"); |
150 CheckKind(tokens, 4, Token::kRPAREN); | 137 CheckKind(tokens, 4, Token::kRPAREN); |
151 CheckLineNumber(tokens, 4, 2); | 138 CheckLineNumber(tokens, 4, 2); |
152 } | 139 } |
153 | 140 |
154 | |
155 static void GreedIsGood() { | 141 static void GreedIsGood() { |
156 // means i++ + j | 142 // means i++ + j |
157 const GrowableTokenStream& tokens = Scan("x=i+++j"); | 143 const GrowableTokenStream& tokens = Scan("x=i+++j"); |
158 | 144 |
159 CheckNumTokens(tokens, 7); | 145 CheckNumTokens(tokens, 7); |
160 CheckIdent(tokens, 0, "x"); | 146 CheckIdent(tokens, 0, "x"); |
161 CheckKind(tokens, 1, Token::kASSIGN); | 147 CheckKind(tokens, 1, Token::kASSIGN); |
162 CheckIdent(tokens, 2, "i"); | 148 CheckIdent(tokens, 2, "i"); |
163 CheckKind(tokens, 3, Token::kINCR); | 149 CheckKind(tokens, 3, Token::kINCR); |
164 CheckKind(tokens, 4, Token::kADD); | 150 CheckKind(tokens, 4, Token::kADD); |
165 CheckIdent(tokens, 5, "j"); | 151 CheckIdent(tokens, 5, "j"); |
166 } | 152 } |
167 | 153 |
168 | |
169 static void StringEscapes() { | 154 static void StringEscapes() { |
170 // sss = "\" \\ \n\r\t \'" | 155 // sss = "\" \\ \n\r\t \'" |
171 const GrowableTokenStream& tokens = | 156 const GrowableTokenStream& tokens = |
172 Scan("sss = \"\\\" \\\\ \\n\\r\\t \\\'\""); | 157 Scan("sss = \"\\\" \\\\ \\n\\r\\t \\\'\""); |
173 | 158 |
174 EXPECT_EQ(4, tokens.length()); | 159 EXPECT_EQ(4, tokens.length()); |
175 CheckIdent(tokens, 0, "sss"); | 160 CheckIdent(tokens, 0, "sss"); |
176 CheckKind(tokens, 1, Token::kASSIGN); | 161 CheckKind(tokens, 1, Token::kASSIGN); |
177 CheckKind(tokens, 2, Token::kSTRING); | 162 CheckKind(tokens, 2, Token::kSTRING); |
178 CheckKind(tokens, 3, Token::kEOS); | 163 CheckKind(tokens, 3, Token::kEOS); |
179 CheckLineNumber(tokens, 2, 1); | 164 CheckLineNumber(tokens, 2, 1); |
180 const char* litchars = (tokens)[2].literal->ToCString(); | 165 const char* litchars = (tokens)[2].literal->ToCString(); |
181 EXPECT_EQ(9, (tokens)[2].literal->Length()); | 166 EXPECT_EQ(9, (tokens)[2].literal->Length()); |
182 | 167 |
183 EXPECT_EQ('"', litchars[0]); | 168 EXPECT_EQ('"', litchars[0]); |
184 EXPECT_EQ(' ', litchars[1]); | 169 EXPECT_EQ(' ', litchars[1]); |
185 EXPECT_EQ('\\', litchars[2]); | 170 EXPECT_EQ('\\', litchars[2]); |
186 EXPECT_EQ('\n', litchars[4]); | 171 EXPECT_EQ('\n', litchars[4]); |
187 EXPECT_EQ('\r', litchars[5]); | 172 EXPECT_EQ('\r', litchars[5]); |
188 EXPECT_EQ('\t', litchars[6]); | 173 EXPECT_EQ('\t', litchars[6]); |
189 EXPECT_EQ('\'', litchars[8]); | 174 EXPECT_EQ('\'', litchars[8]); |
190 } | 175 } |
191 | 176 |
192 | |
193 static void InvalidStringEscapes() { | 177 static void InvalidStringEscapes() { |
194 const GrowableTokenStream& out_of_range_low = Scan("\"\\u{110000}\""); | 178 const GrowableTokenStream& out_of_range_low = Scan("\"\\u{110000}\""); |
195 EXPECT_EQ(2, out_of_range_low.length()); | 179 EXPECT_EQ(2, out_of_range_low.length()); |
196 CheckKind(out_of_range_low, 0, Token::kERROR); | 180 CheckKind(out_of_range_low, 0, Token::kERROR); |
197 EXPECT(out_of_range_low[0].literal->Equals("invalid code point")); | 181 EXPECT(out_of_range_low[0].literal->Equals("invalid code point")); |
198 CheckKind(out_of_range_low, 1, Token::kEOS); | 182 CheckKind(out_of_range_low, 1, Token::kEOS); |
199 | 183 |
200 const GrowableTokenStream& out_of_range_high = Scan("\"\\u{FFFFFF}\""); | 184 const GrowableTokenStream& out_of_range_high = Scan("\"\\u{FFFFFF}\""); |
201 EXPECT_EQ(2, out_of_range_high.length()); | 185 EXPECT_EQ(2, out_of_range_high.length()); |
202 CheckKind(out_of_range_high, 0, Token::kERROR); | 186 CheckKind(out_of_range_high, 0, Token::kERROR); |
203 EXPECT(out_of_range_high[0].literal->Equals("invalid code point")); | 187 EXPECT(out_of_range_high[0].literal->Equals("invalid code point")); |
204 CheckKind(out_of_range_high, 1, Token::kEOS); | 188 CheckKind(out_of_range_high, 1, Token::kEOS); |
205 } | 189 } |
206 | 190 |
207 | |
208 static void RawString() { | 191 static void RawString() { |
209 // rs = @"\' \\" | 192 // rs = @"\' \\" |
210 const GrowableTokenStream& tokens = Scan("rs = r\"\\\' \\\\\""); | 193 const GrowableTokenStream& tokens = Scan("rs = r\"\\\' \\\\\""); |
211 | 194 |
212 EXPECT_EQ(4, tokens.length()); | 195 EXPECT_EQ(4, tokens.length()); |
213 CheckIdent(tokens, 0, "rs"); | 196 CheckIdent(tokens, 0, "rs"); |
214 CheckKind(tokens, 1, Token::kASSIGN); | 197 CheckKind(tokens, 1, Token::kASSIGN); |
215 CheckKind(tokens, 2, Token::kSTRING); | 198 CheckKind(tokens, 2, Token::kSTRING); |
216 CheckKind(tokens, 3, Token::kEOS); | 199 CheckKind(tokens, 3, Token::kEOS); |
217 CheckLineNumber(tokens, 2, 1); | 200 CheckLineNumber(tokens, 2, 1); |
218 const char* litchars = (tokens)[2].literal->ToCString(); | 201 const char* litchars = (tokens)[2].literal->ToCString(); |
219 EXPECT_EQ(5, (tokens)[2].literal->Length()); | 202 EXPECT_EQ(5, (tokens)[2].literal->Length()); |
220 | 203 |
221 EXPECT_EQ('\\', litchars[0]); | 204 EXPECT_EQ('\\', litchars[0]); |
222 EXPECT_EQ('\'', litchars[1]); | 205 EXPECT_EQ('\'', litchars[1]); |
223 EXPECT_EQ(' ', litchars[2]); | 206 EXPECT_EQ(' ', litchars[2]); |
224 EXPECT_EQ('\\', litchars[3]); | 207 EXPECT_EQ('\\', litchars[3]); |
225 EXPECT_EQ('\\', litchars[4]); | 208 EXPECT_EQ('\\', litchars[4]); |
226 } | 209 } |
227 | 210 |
228 | |
229 static void MultilineString() { | 211 static void MultilineString() { |
230 // |mls = ''' | 212 // |mls = ''' |
231 // |1' x | 213 // |1' x |
232 // |2'''; | 214 // |2'''; |
233 const GrowableTokenStream& tokens = Scan("mls = '''\n1' x\n2''';"); | 215 const GrowableTokenStream& tokens = Scan("mls = '''\n1' x\n2''';"); |
234 | 216 |
235 EXPECT_EQ(7, tokens.length()); | 217 EXPECT_EQ(7, tokens.length()); |
236 CheckIdent(tokens, 0, "mls"); | 218 CheckIdent(tokens, 0, "mls"); |
237 CheckKind(tokens, 1, Token::kASSIGN); | 219 CheckKind(tokens, 1, Token::kASSIGN); |
238 CheckKind(tokens, 2, Token::kSTRING); | 220 CheckKind(tokens, 2, Token::kSTRING); |
239 CheckKind(tokens, 3, Token::kNEWLINE); | 221 CheckKind(tokens, 3, Token::kNEWLINE); |
240 CheckKind(tokens, 4, Token::kNEWLINE); | 222 CheckKind(tokens, 4, Token::kNEWLINE); |
241 CheckKind(tokens, 5, Token::kSEMICOLON); | 223 CheckKind(tokens, 5, Token::kSEMICOLON); |
242 CheckKind(tokens, 6, Token::kEOS); | 224 CheckKind(tokens, 6, Token::kEOS); |
243 CheckLineNumber(tokens, 0, 1); | 225 CheckLineNumber(tokens, 0, 1); |
244 CheckLineNumber(tokens, 5, 3); // Semicolon is on line 3. | 226 CheckLineNumber(tokens, 5, 3); // Semicolon is on line 3. |
245 const char* litchars = (tokens)[2].literal->ToCString(); | 227 const char* litchars = (tokens)[2].literal->ToCString(); |
246 EXPECT_EQ(6, (tokens)[2].literal->Length()); | 228 EXPECT_EQ(6, (tokens)[2].literal->Length()); |
247 | 229 |
248 EXPECT_EQ('1', litchars[0]); // First newline is dropped. | 230 EXPECT_EQ('1', litchars[0]); // First newline is dropped. |
249 EXPECT_EQ('\'', litchars[1]); | 231 EXPECT_EQ('\'', litchars[1]); |
250 EXPECT_EQ(' ', litchars[2]); | 232 EXPECT_EQ(' ', litchars[2]); |
251 EXPECT_EQ('x', litchars[3]); | 233 EXPECT_EQ('x', litchars[3]); |
252 EXPECT_EQ('\n', litchars[4]); | 234 EXPECT_EQ('\n', litchars[4]); |
253 EXPECT_EQ('2', litchars[5]); | 235 EXPECT_EQ('2', litchars[5]); |
254 } | 236 } |
255 | 237 |
256 | |
257 static void EmptyString() { | 238 static void EmptyString() { |
258 // es = ""; | 239 // es = ""; |
259 const GrowableTokenStream& tokens = Scan("es = \"\";"); | 240 const GrowableTokenStream& tokens = Scan("es = \"\";"); |
260 | 241 |
261 EXPECT_EQ(5, tokens.length()); | 242 EXPECT_EQ(5, tokens.length()); |
262 CheckIdent(tokens, 0, "es"); | 243 CheckIdent(tokens, 0, "es"); |
263 CheckKind(tokens, 1, Token::kASSIGN); | 244 CheckKind(tokens, 1, Token::kASSIGN); |
264 CheckKind(tokens, 2, Token::kSTRING); | 245 CheckKind(tokens, 2, Token::kSTRING); |
265 CheckKind(tokens, 3, Token::kSEMICOLON); | 246 CheckKind(tokens, 3, Token::kSEMICOLON); |
266 CheckKind(tokens, 4, Token::kEOS); | 247 CheckKind(tokens, 4, Token::kEOS); |
267 EXPECT_EQ(0, (tokens)[2].literal->Length()); | 248 EXPECT_EQ(0, (tokens)[2].literal->Length()); |
268 } | 249 } |
269 | 250 |
270 static void EmptyMultilineString() { | 251 static void EmptyMultilineString() { |
271 // es = """"""; | 252 // es = """"""; |
272 const GrowableTokenStream& tokens = Scan("es = \"\"\"\"\"\";"); | 253 const GrowableTokenStream& tokens = Scan("es = \"\"\"\"\"\";"); |
273 | 254 |
274 EXPECT_EQ(5, tokens.length()); | 255 EXPECT_EQ(5, tokens.length()); |
275 CheckIdent(tokens, 0, "es"); | 256 CheckIdent(tokens, 0, "es"); |
276 CheckKind(tokens, 1, Token::kASSIGN); | 257 CheckKind(tokens, 1, Token::kASSIGN); |
277 CheckKind(tokens, 2, Token::kSTRING); | 258 CheckKind(tokens, 2, Token::kSTRING); |
278 CheckKind(tokens, 3, Token::kSEMICOLON); | 259 CheckKind(tokens, 3, Token::kSEMICOLON); |
279 CheckKind(tokens, 4, Token::kEOS); | 260 CheckKind(tokens, 4, Token::kEOS); |
280 EXPECT_EQ(0, (tokens)[2].literal->Length()); | 261 EXPECT_EQ(0, (tokens)[2].literal->Length()); |
281 } | 262 } |
282 | 263 |
283 | |
284 static void NumberLiteral() { | 264 static void NumberLiteral() { |
285 const GrowableTokenStream& tokens = Scan("5 0x5d 0.3 0.33 1E+12 .42 +5"); | 265 const GrowableTokenStream& tokens = Scan("5 0x5d 0.3 0.33 1E+12 .42 +5"); |
286 | 266 |
287 CheckKind(tokens, 0, Token::kINTEGER); | 267 CheckKind(tokens, 0, Token::kINTEGER); |
288 CheckKind(tokens, 1, Token::kINTEGER); | 268 CheckKind(tokens, 1, Token::kINTEGER); |
289 CheckKind(tokens, 2, Token::kDOUBLE); | 269 CheckKind(tokens, 2, Token::kDOUBLE); |
290 CheckKind(tokens, 3, Token::kDOUBLE); | 270 CheckKind(tokens, 3, Token::kDOUBLE); |
291 CheckKind(tokens, 4, Token::kDOUBLE); | 271 CheckKind(tokens, 4, Token::kDOUBLE); |
292 CheckKind(tokens, 5, Token::kDOUBLE); | 272 CheckKind(tokens, 5, Token::kDOUBLE); |
293 CheckKind(tokens, 6, Token::kADD); | 273 CheckKind(tokens, 6, Token::kADD); |
294 CheckKind(tokens, 7, Token::kINTEGER); | 274 CheckKind(tokens, 7, Token::kINTEGER); |
295 CheckKind(tokens, 8, Token::kEOS); | 275 CheckKind(tokens, 8, Token::kEOS); |
296 } | 276 } |
297 | 277 |
298 | |
299 static void ScanLargeText() { | 278 static void ScanLargeText() { |
300 const char* dart_source = | 279 const char* dart_source = |
301 "// This source is not meant to be valid Dart code. The text is used to" | 280 "// This source is not meant to be valid Dart code. The text is used to" |
302 "// test the Dart scanner." | 281 "// test the Dart scanner." |
303 "" | 282 "" |
304 "// Cartesian point implementation." | 283 "// Cartesian point implementation." |
305 "class Point {" | 284 "class Point {" |
306 "" | 285 "" |
307 " // Constructor" | 286 " // Constructor" |
308 " Point(Number x, Number y) : x(x), y(y) { }" | 287 " Point(Number x, Number y) : x(x), y(y) { }" |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
352 " for (int i = 0; i < this.length; i++) {" | 331 " for (int i = 0; i < this.length; i++) {" |
353 " f(this[i]);" | 332 " f(this[i]);" |
354 " }" | 333 " }" |
355 "}" | 334 "}" |
356 "" | 335 "" |
357 "" | 336 "" |
358 "j!==!iffy // means j !== !iffy"; | 337 "j!==!iffy // means j !== !iffy"; |
359 Scan(dart_source); | 338 Scan(dart_source); |
360 } | 339 } |
361 | 340 |
362 | |
363 void InvalidText() { | 341 void InvalidText() { |
364 const GrowableTokenStream& tokens = Scan("\\"); | 342 const GrowableTokenStream& tokens = Scan("\\"); |
365 | 343 |
366 EXPECT_EQ(2, tokens.length()); | 344 EXPECT_EQ(2, tokens.length()); |
367 CheckKind(tokens, 0, Token::kERROR); | 345 CheckKind(tokens, 0, Token::kERROR); |
368 CheckKind(tokens, 1, Token::kEOS); | 346 CheckKind(tokens, 1, Token::kEOS); |
369 } | 347 } |
370 | 348 |
371 | |
372 void NewlinesTest() { | 349 void NewlinesTest() { |
373 const char* source = | 350 const char* source = |
374 "var es = /* a\n" | 351 "var es = /* a\n" |
375 " b\n" | 352 " b\n" |
376 " */ \"\"\"\n" | 353 " */ \"\"\"\n" |
377 "c\n" | 354 "c\n" |
378 "d\n" | 355 "d\n" |
379 "\"\"\";"; | 356 "\"\"\";"; |
380 | 357 |
381 const GrowableTokenStream& tokens = Scan(source); | 358 const GrowableTokenStream& tokens = Scan(source); |
(...skipping 12 matching lines...) Expand all Loading... |
394 CheckKind(tokens, 10, Token::kEOS); | 371 CheckKind(tokens, 10, Token::kEOS); |
395 | 372 |
396 EXPECT_EQ(4, (tokens)[5].literal->Length()); | 373 EXPECT_EQ(4, (tokens)[5].literal->Length()); |
397 const char* litchars = (tokens)[5].literal->ToCString(); | 374 const char* litchars = (tokens)[5].literal->ToCString(); |
398 EXPECT_EQ('c', litchars[0]); // First newline is dropped. | 375 EXPECT_EQ('c', litchars[0]); // First newline is dropped. |
399 EXPECT_EQ('\n', litchars[1]); | 376 EXPECT_EQ('\n', litchars[1]); |
400 EXPECT_EQ('d', litchars[2]); | 377 EXPECT_EQ('d', litchars[2]); |
401 EXPECT_EQ('\n', litchars[3]); | 378 EXPECT_EQ('\n', litchars[3]); |
402 } | 379 } |
403 | 380 |
404 | |
405 TEST_CASE(Scanner_Test) { | 381 TEST_CASE(Scanner_Test) { |
406 ScanLargeText(); | 382 ScanLargeText(); |
407 | 383 |
408 BoringTest(); | 384 BoringTest(); |
409 CommentTest(); | 385 CommentTest(); |
410 GreedIsGood(); | 386 GreedIsGood(); |
411 StringEscapes(); | 387 StringEscapes(); |
412 InvalidStringEscapes(); | 388 InvalidStringEscapes(); |
413 RawString(); | 389 RawString(); |
414 MultilineString(); | 390 MultilineString(); |
415 EmptyString(); | 391 EmptyString(); |
416 EmptyMultilineString(); | 392 EmptyMultilineString(); |
417 NumberLiteral(); | 393 NumberLiteral(); |
418 InvalidText(); | 394 InvalidText(); |
419 NewlinesTest(); | 395 NewlinesTest(); |
420 } | 396 } |
421 | 397 |
422 } // namespace dart | 398 } // namespace dart |
OLD | NEW |