| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/scanner.h" |
| 5 #include "platform/assert.h" | 6 #include "platform/assert.h" |
| 6 #include "vm/os.h" | 7 #include "vm/os.h" |
| 7 #include "vm/scanner.h" | |
| 8 #include "vm/token.h" | 8 #include "vm/token.h" |
| 9 #include "vm/unit_test.h" | 9 #include "vm/unit_test.h" |
| 10 | 10 |
| 11 namespace dart { | 11 namespace dart { |
| 12 | 12 |
| 13 | |
| 14 typedef ZoneGrowableArray<Scanner::TokenDescriptor> GrowableTokenStream; | 13 typedef ZoneGrowableArray<Scanner::TokenDescriptor> GrowableTokenStream; |
| 15 | 14 |
| 16 | |
| 17 static void LogTokenDesc(Scanner::TokenDescriptor token) { | 15 static void LogTokenDesc(Scanner::TokenDescriptor token) { |
| 18 OS::Print("pos %2d:%d-%d token %s ", token.position.line, | 16 OS::Print("pos %2d:%d-%d token %s ", token.position.line, |
| 19 token.position.column, token.position.column, | 17 token.position.column, token.position.column, |
| 20 Token::Name(token.kind)); | 18 Token::Name(token.kind)); |
| 21 if (token.literal != NULL) { | 19 if (token.literal != NULL) { |
| 22 OS::Print("%s", token.literal->ToCString()); | 20 OS::Print("%s", token.literal->ToCString()); |
| 23 } | 21 } |
| 24 OS::Print("\n"); | 22 OS::Print("\n"); |
| 25 } | 23 } |
| 26 | 24 |
| 27 | |
| 28 static void LogTokenStream(const GrowableTokenStream& token_stream) { | 25 static void LogTokenStream(const GrowableTokenStream& token_stream) { |
| 29 int token_index = 0; | 26 int token_index = 0; |
| 30 EXPECT_GT(token_stream.length(), 0); | 27 EXPECT_GT(token_stream.length(), 0); |
| 31 while (token_index < token_stream.length()) { | 28 while (token_index < token_stream.length()) { |
| 32 LogTokenDesc(token_stream[token_index]); | 29 LogTokenDesc(token_stream[token_index]); |
| 33 ASSERT(token_stream[token_index].kind != Token::kILLEGAL); | 30 ASSERT(token_stream[token_index].kind != Token::kILLEGAL); |
| 34 token_index++; | 31 token_index++; |
| 35 } | 32 } |
| 36 printf("%d tokens in stream.\n", token_index); | 33 printf("%d tokens in stream.\n", token_index); |
| 37 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); | 34 EXPECT_EQ(token_stream.Last().kind, Token::kEOS); |
| 38 } | 35 } |
| 39 | 36 |
| 40 | |
| 41 static void CheckKind(const GrowableTokenStream& token_stream, | 37 static void CheckKind(const GrowableTokenStream& token_stream, |
| 42 int index, | 38 int index, |
| 43 Token::Kind kind) { | 39 Token::Kind kind) { |
| 44 if (token_stream[index].kind != kind) { | 40 if (token_stream[index].kind != kind) { |
| 45 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, | 41 OS::PrintErr("Token %d: expected kind %s but got %s\n", index, |
| 46 Token::Name(kind), Token::Name(token_stream[index].kind)); | 42 Token::Name(kind), Token::Name(token_stream[index].kind)); |
| 47 } | 43 } |
| 48 EXPECT_EQ(kind, token_stream[index].kind); | 44 EXPECT_EQ(kind, token_stream[index].kind); |
| 49 } | 45 } |
| 50 | 46 |
| 51 | |
| 52 static void CheckLiteral(const GrowableTokenStream& token_stream, | 47 static void CheckLiteral(const GrowableTokenStream& token_stream, |
| 53 int index, | 48 int index, |
| 54 const char* literal) { | 49 const char* literal) { |
| 55 if (token_stream[index].literal == NULL) { | 50 if (token_stream[index].literal == NULL) { |
| 56 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", index, | 51 OS::PrintErr("Token %d: expected literal \"%s\" but got nothing\n", index, |
| 57 literal); | 52 literal); |
| 58 } else if (strcmp(literal, token_stream[index].literal->ToCString())) { | 53 } else if (strcmp(literal, token_stream[index].literal->ToCString())) { |
| 59 OS::PrintErr("Token %d: expected literal \"%s\" but got \"%s\"\n", index, | 54 OS::PrintErr("Token %d: expected literal \"%s\" but got \"%s\"\n", index, |
| 60 literal, token_stream[index].literal->ToCString()); | 55 literal, token_stream[index].literal->ToCString()); |
| 61 } | 56 } |
| 62 } | 57 } |
| 63 | 58 |
| 64 | |
| 65 static void CheckIdent(const GrowableTokenStream& token_stream, | 59 static void CheckIdent(const GrowableTokenStream& token_stream, |
| 66 int index, | 60 int index, |
| 67 const char* literal) { | 61 const char* literal) { |
| 68 CheckKind(token_stream, index, Token::kIDENT); | 62 CheckKind(token_stream, index, Token::kIDENT); |
| 69 CheckLiteral(token_stream, index, literal); | 63 CheckLiteral(token_stream, index, literal); |
| 70 } | 64 } |
| 71 | 65 |
| 72 | |
| 73 static void CheckInteger(const GrowableTokenStream& token_stream, | 66 static void CheckInteger(const GrowableTokenStream& token_stream, |
| 74 int index, | 67 int index, |
| 75 const char* literal) { | 68 const char* literal) { |
| 76 CheckKind(token_stream, index, Token::kINTEGER); | 69 CheckKind(token_stream, index, Token::kINTEGER); |
| 77 CheckLiteral(token_stream, index, literal); | 70 CheckLiteral(token_stream, index, literal); |
| 78 } | 71 } |
| 79 | 72 |
| 80 | |
| 81 static void CheckLineNumber(const GrowableTokenStream& token_stream, | 73 static void CheckLineNumber(const GrowableTokenStream& token_stream, |
| 82 int index, | 74 int index, |
| 83 int line_number) { | 75 int line_number) { |
| 84 if (token_stream[index].position.line != line_number) { | 76 if (token_stream[index].position.line != line_number) { |
| 85 OS::PrintErr("Token %d: expected line number %d but got %d\n", index, | 77 OS::PrintErr("Token %d: expected line number %d but got %d\n", index, |
| 86 line_number, token_stream[index].position.line); | 78 line_number, token_stream[index].position.line); |
| 87 } | 79 } |
| 88 } | 80 } |
| 89 | 81 |
| 90 | |
| 91 static void CheckNumTokens(const GrowableTokenStream& token_stream, int index) { | 82 static void CheckNumTokens(const GrowableTokenStream& token_stream, int index) { |
| 92 if (token_stream.length() != index) { | 83 if (token_stream.length() != index) { |
| 93 OS::PrintErr("Expected %d tokens but got only %" Pd ".\n", index, | 84 OS::PrintErr("Expected %d tokens but got only %" Pd ".\n", index, |
| 94 token_stream.length()); | 85 token_stream.length()); |
| 95 } | 86 } |
| 96 } | 87 } |
| 97 | 88 |
| 98 | |
| 99 class Collector : public Scanner::TokenCollector { | 89 class Collector : public Scanner::TokenCollector { |
| 100 public: | 90 public: |
| 101 explicit Collector(GrowableTokenStream* ts) : ts_(ts) {} | 91 explicit Collector(GrowableTokenStream* ts) : ts_(ts) {} |
| 102 virtual ~Collector() {} | 92 virtual ~Collector() {} |
| 103 | 93 |
| 104 virtual void AddToken(const Scanner::TokenDescriptor& token) { | 94 virtual void AddToken(const Scanner::TokenDescriptor& token) { |
| 105 ts_->Add(token); | 95 ts_->Add(token); |
| 106 } | 96 } |
| 107 | 97 |
| 108 private: | 98 private: |
| 109 GrowableTokenStream* ts_; | 99 GrowableTokenStream* ts_; |
| 110 }; | 100 }; |
| 111 | 101 |
| 112 | |
| 113 static const GrowableTokenStream& Scan(const char* source) { | 102 static const GrowableTokenStream& Scan(const char* source) { |
| 114 OS::Print("\nScanning: <%s>\n", source); | 103 OS::Print("\nScanning: <%s>\n", source); |
| 115 | 104 |
| 116 Scanner scanner(String::Handle(String::New(source)), | 105 Scanner scanner(String::Handle(String::New(source)), |
| 117 String::Handle(String::New(""))); | 106 String::Handle(String::New(""))); |
| 118 GrowableTokenStream* tokens = new GrowableTokenStream(128); | 107 GrowableTokenStream* tokens = new GrowableTokenStream(128); |
| 119 Collector collector(tokens); | 108 Collector collector(tokens); |
| 120 | 109 |
| 121 scanner.ScanAll(&collector); | 110 scanner.ScanAll(&collector); |
| 122 LogTokenStream(*tokens); | 111 LogTokenStream(*tokens); |
| 123 return *tokens; | 112 return *tokens; |
| 124 } | 113 } |
| 125 | 114 |
| 126 | |
| 127 static void BoringTest() { | 115 static void BoringTest() { |
| 128 const GrowableTokenStream& tokens = Scan("x = iffy++;"); | 116 const GrowableTokenStream& tokens = Scan("x = iffy++;"); |
| 129 | 117 |
| 130 CheckNumTokens(tokens, 6); | 118 CheckNumTokens(tokens, 6); |
| 131 CheckIdent(tokens, 0, "x"); | 119 CheckIdent(tokens, 0, "x"); |
| 132 CheckKind(tokens, 1, Token::kASSIGN); | 120 CheckKind(tokens, 1, Token::kASSIGN); |
| 133 CheckIdent(tokens, 2, "iffy"); | 121 CheckIdent(tokens, 2, "iffy"); |
| 134 CheckKind(tokens, 3, Token::kINCR); | 122 CheckKind(tokens, 3, Token::kINCR); |
| 135 CheckKind(tokens, 4, Token::kSEMICOLON); | 123 CheckKind(tokens, 4, Token::kSEMICOLON); |
| 136 } | 124 } |
| 137 | 125 |
| 138 | |
| 139 static void CommentTest() { | 126 static void CommentTest() { |
| 140 const GrowableTokenStream& tokens = Scan( | 127 const GrowableTokenStream& tokens = Scan( |
| 141 "Foo( /*block \n" | 128 "Foo( /*block \n" |
| 142 "comment*/ 0xff) // line comment;"); | 129 "comment*/ 0xff) // line comment;"); |
| 143 | 130 |
| 144 CheckNumTokens(tokens, 6); | 131 CheckNumTokens(tokens, 6); |
| 145 CheckIdent(tokens, 0, "Foo"); | 132 CheckIdent(tokens, 0, "Foo"); |
| 146 CheckLineNumber(tokens, 0, 1); | 133 CheckLineNumber(tokens, 0, 1); |
| 147 CheckKind(tokens, 1, Token::kLPAREN); | 134 CheckKind(tokens, 1, Token::kLPAREN); |
| 148 CheckKind(tokens, 2, Token::kNEWLINE); | 135 CheckKind(tokens, 2, Token::kNEWLINE); |
| 149 CheckInteger(tokens, 3, "0xff"); | 136 CheckInteger(tokens, 3, "0xff"); |
| 150 CheckKind(tokens, 4, Token::kRPAREN); | 137 CheckKind(tokens, 4, Token::kRPAREN); |
| 151 CheckLineNumber(tokens, 4, 2); | 138 CheckLineNumber(tokens, 4, 2); |
| 152 } | 139 } |
| 153 | 140 |
| 154 | |
| 155 static void GreedIsGood() { | 141 static void GreedIsGood() { |
| 156 // means i++ + j | 142 // means i++ + j |
| 157 const GrowableTokenStream& tokens = Scan("x=i+++j"); | 143 const GrowableTokenStream& tokens = Scan("x=i+++j"); |
| 158 | 144 |
| 159 CheckNumTokens(tokens, 7); | 145 CheckNumTokens(tokens, 7); |
| 160 CheckIdent(tokens, 0, "x"); | 146 CheckIdent(tokens, 0, "x"); |
| 161 CheckKind(tokens, 1, Token::kASSIGN); | 147 CheckKind(tokens, 1, Token::kASSIGN); |
| 162 CheckIdent(tokens, 2, "i"); | 148 CheckIdent(tokens, 2, "i"); |
| 163 CheckKind(tokens, 3, Token::kINCR); | 149 CheckKind(tokens, 3, Token::kINCR); |
| 164 CheckKind(tokens, 4, Token::kADD); | 150 CheckKind(tokens, 4, Token::kADD); |
| 165 CheckIdent(tokens, 5, "j"); | 151 CheckIdent(tokens, 5, "j"); |
| 166 } | 152 } |
| 167 | 153 |
| 168 | |
| 169 static void StringEscapes() { | 154 static void StringEscapes() { |
| 170 // sss = "\" \\ \n\r\t \'" | 155 // sss = "\" \\ \n\r\t \'" |
| 171 const GrowableTokenStream& tokens = | 156 const GrowableTokenStream& tokens = |
| 172 Scan("sss = \"\\\" \\\\ \\n\\r\\t \\\'\""); | 157 Scan("sss = \"\\\" \\\\ \\n\\r\\t \\\'\""); |
| 173 | 158 |
| 174 EXPECT_EQ(4, tokens.length()); | 159 EXPECT_EQ(4, tokens.length()); |
| 175 CheckIdent(tokens, 0, "sss"); | 160 CheckIdent(tokens, 0, "sss"); |
| 176 CheckKind(tokens, 1, Token::kASSIGN); | 161 CheckKind(tokens, 1, Token::kASSIGN); |
| 177 CheckKind(tokens, 2, Token::kSTRING); | 162 CheckKind(tokens, 2, Token::kSTRING); |
| 178 CheckKind(tokens, 3, Token::kEOS); | 163 CheckKind(tokens, 3, Token::kEOS); |
| 179 CheckLineNumber(tokens, 2, 1); | 164 CheckLineNumber(tokens, 2, 1); |
| 180 const char* litchars = (tokens)[2].literal->ToCString(); | 165 const char* litchars = (tokens)[2].literal->ToCString(); |
| 181 EXPECT_EQ(9, (tokens)[2].literal->Length()); | 166 EXPECT_EQ(9, (tokens)[2].literal->Length()); |
| 182 | 167 |
| 183 EXPECT_EQ('"', litchars[0]); | 168 EXPECT_EQ('"', litchars[0]); |
| 184 EXPECT_EQ(' ', litchars[1]); | 169 EXPECT_EQ(' ', litchars[1]); |
| 185 EXPECT_EQ('\\', litchars[2]); | 170 EXPECT_EQ('\\', litchars[2]); |
| 186 EXPECT_EQ('\n', litchars[4]); | 171 EXPECT_EQ('\n', litchars[4]); |
| 187 EXPECT_EQ('\r', litchars[5]); | 172 EXPECT_EQ('\r', litchars[5]); |
| 188 EXPECT_EQ('\t', litchars[6]); | 173 EXPECT_EQ('\t', litchars[6]); |
| 189 EXPECT_EQ('\'', litchars[8]); | 174 EXPECT_EQ('\'', litchars[8]); |
| 190 } | 175 } |
| 191 | 176 |
| 192 | |
| 193 static void InvalidStringEscapes() { | 177 static void InvalidStringEscapes() { |
| 194 const GrowableTokenStream& out_of_range_low = Scan("\"\\u{110000}\""); | 178 const GrowableTokenStream& out_of_range_low = Scan("\"\\u{110000}\""); |
| 195 EXPECT_EQ(2, out_of_range_low.length()); | 179 EXPECT_EQ(2, out_of_range_low.length()); |
| 196 CheckKind(out_of_range_low, 0, Token::kERROR); | 180 CheckKind(out_of_range_low, 0, Token::kERROR); |
| 197 EXPECT(out_of_range_low[0].literal->Equals("invalid code point")); | 181 EXPECT(out_of_range_low[0].literal->Equals("invalid code point")); |
| 198 CheckKind(out_of_range_low, 1, Token::kEOS); | 182 CheckKind(out_of_range_low, 1, Token::kEOS); |
| 199 | 183 |
| 200 const GrowableTokenStream& out_of_range_high = Scan("\"\\u{FFFFFF}\""); | 184 const GrowableTokenStream& out_of_range_high = Scan("\"\\u{FFFFFF}\""); |
| 201 EXPECT_EQ(2, out_of_range_high.length()); | 185 EXPECT_EQ(2, out_of_range_high.length()); |
| 202 CheckKind(out_of_range_high, 0, Token::kERROR); | 186 CheckKind(out_of_range_high, 0, Token::kERROR); |
| 203 EXPECT(out_of_range_high[0].literal->Equals("invalid code point")); | 187 EXPECT(out_of_range_high[0].literal->Equals("invalid code point")); |
| 204 CheckKind(out_of_range_high, 1, Token::kEOS); | 188 CheckKind(out_of_range_high, 1, Token::kEOS); |
| 205 } | 189 } |
| 206 | 190 |
| 207 | |
| 208 static void RawString() { | 191 static void RawString() { |
| 209 // rs = @"\' \\" | 192 // rs = @"\' \\" |
| 210 const GrowableTokenStream& tokens = Scan("rs = r\"\\\' \\\\\""); | 193 const GrowableTokenStream& tokens = Scan("rs = r\"\\\' \\\\\""); |
| 211 | 194 |
| 212 EXPECT_EQ(4, tokens.length()); | 195 EXPECT_EQ(4, tokens.length()); |
| 213 CheckIdent(tokens, 0, "rs"); | 196 CheckIdent(tokens, 0, "rs"); |
| 214 CheckKind(tokens, 1, Token::kASSIGN); | 197 CheckKind(tokens, 1, Token::kASSIGN); |
| 215 CheckKind(tokens, 2, Token::kSTRING); | 198 CheckKind(tokens, 2, Token::kSTRING); |
| 216 CheckKind(tokens, 3, Token::kEOS); | 199 CheckKind(tokens, 3, Token::kEOS); |
| 217 CheckLineNumber(tokens, 2, 1); | 200 CheckLineNumber(tokens, 2, 1); |
| 218 const char* litchars = (tokens)[2].literal->ToCString(); | 201 const char* litchars = (tokens)[2].literal->ToCString(); |
| 219 EXPECT_EQ(5, (tokens)[2].literal->Length()); | 202 EXPECT_EQ(5, (tokens)[2].literal->Length()); |
| 220 | 203 |
| 221 EXPECT_EQ('\\', litchars[0]); | 204 EXPECT_EQ('\\', litchars[0]); |
| 222 EXPECT_EQ('\'', litchars[1]); | 205 EXPECT_EQ('\'', litchars[1]); |
| 223 EXPECT_EQ(' ', litchars[2]); | 206 EXPECT_EQ(' ', litchars[2]); |
| 224 EXPECT_EQ('\\', litchars[3]); | 207 EXPECT_EQ('\\', litchars[3]); |
| 225 EXPECT_EQ('\\', litchars[4]); | 208 EXPECT_EQ('\\', litchars[4]); |
| 226 } | 209 } |
| 227 | 210 |
| 228 | |
| 229 static void MultilineString() { | 211 static void MultilineString() { |
| 230 // |mls = ''' | 212 // |mls = ''' |
| 231 // |1' x | 213 // |1' x |
| 232 // |2'''; | 214 // |2'''; |
| 233 const GrowableTokenStream& tokens = Scan("mls = '''\n1' x\n2''';"); | 215 const GrowableTokenStream& tokens = Scan("mls = '''\n1' x\n2''';"); |
| 234 | 216 |
| 235 EXPECT_EQ(7, tokens.length()); | 217 EXPECT_EQ(7, tokens.length()); |
| 236 CheckIdent(tokens, 0, "mls"); | 218 CheckIdent(tokens, 0, "mls"); |
| 237 CheckKind(tokens, 1, Token::kASSIGN); | 219 CheckKind(tokens, 1, Token::kASSIGN); |
| 238 CheckKind(tokens, 2, Token::kSTRING); | 220 CheckKind(tokens, 2, Token::kSTRING); |
| 239 CheckKind(tokens, 3, Token::kNEWLINE); | 221 CheckKind(tokens, 3, Token::kNEWLINE); |
| 240 CheckKind(tokens, 4, Token::kNEWLINE); | 222 CheckKind(tokens, 4, Token::kNEWLINE); |
| 241 CheckKind(tokens, 5, Token::kSEMICOLON); | 223 CheckKind(tokens, 5, Token::kSEMICOLON); |
| 242 CheckKind(tokens, 6, Token::kEOS); | 224 CheckKind(tokens, 6, Token::kEOS); |
| 243 CheckLineNumber(tokens, 0, 1); | 225 CheckLineNumber(tokens, 0, 1); |
| 244 CheckLineNumber(tokens, 5, 3); // Semicolon is on line 3. | 226 CheckLineNumber(tokens, 5, 3); // Semicolon is on line 3. |
| 245 const char* litchars = (tokens)[2].literal->ToCString(); | 227 const char* litchars = (tokens)[2].literal->ToCString(); |
| 246 EXPECT_EQ(6, (tokens)[2].literal->Length()); | 228 EXPECT_EQ(6, (tokens)[2].literal->Length()); |
| 247 | 229 |
| 248 EXPECT_EQ('1', litchars[0]); // First newline is dropped. | 230 EXPECT_EQ('1', litchars[0]); // First newline is dropped. |
| 249 EXPECT_EQ('\'', litchars[1]); | 231 EXPECT_EQ('\'', litchars[1]); |
| 250 EXPECT_EQ(' ', litchars[2]); | 232 EXPECT_EQ(' ', litchars[2]); |
| 251 EXPECT_EQ('x', litchars[3]); | 233 EXPECT_EQ('x', litchars[3]); |
| 252 EXPECT_EQ('\n', litchars[4]); | 234 EXPECT_EQ('\n', litchars[4]); |
| 253 EXPECT_EQ('2', litchars[5]); | 235 EXPECT_EQ('2', litchars[5]); |
| 254 } | 236 } |
| 255 | 237 |
| 256 | |
| 257 static void EmptyString() { | 238 static void EmptyString() { |
| 258 // es = ""; | 239 // es = ""; |
| 259 const GrowableTokenStream& tokens = Scan("es = \"\";"); | 240 const GrowableTokenStream& tokens = Scan("es = \"\";"); |
| 260 | 241 |
| 261 EXPECT_EQ(5, tokens.length()); | 242 EXPECT_EQ(5, tokens.length()); |
| 262 CheckIdent(tokens, 0, "es"); | 243 CheckIdent(tokens, 0, "es"); |
| 263 CheckKind(tokens, 1, Token::kASSIGN); | 244 CheckKind(tokens, 1, Token::kASSIGN); |
| 264 CheckKind(tokens, 2, Token::kSTRING); | 245 CheckKind(tokens, 2, Token::kSTRING); |
| 265 CheckKind(tokens, 3, Token::kSEMICOLON); | 246 CheckKind(tokens, 3, Token::kSEMICOLON); |
| 266 CheckKind(tokens, 4, Token::kEOS); | 247 CheckKind(tokens, 4, Token::kEOS); |
| 267 EXPECT_EQ(0, (tokens)[2].literal->Length()); | 248 EXPECT_EQ(0, (tokens)[2].literal->Length()); |
| 268 } | 249 } |
| 269 | 250 |
| 270 static void EmptyMultilineString() { | 251 static void EmptyMultilineString() { |
| 271 // es = """"""; | 252 // es = """"""; |
| 272 const GrowableTokenStream& tokens = Scan("es = \"\"\"\"\"\";"); | 253 const GrowableTokenStream& tokens = Scan("es = \"\"\"\"\"\";"); |
| 273 | 254 |
| 274 EXPECT_EQ(5, tokens.length()); | 255 EXPECT_EQ(5, tokens.length()); |
| 275 CheckIdent(tokens, 0, "es"); | 256 CheckIdent(tokens, 0, "es"); |
| 276 CheckKind(tokens, 1, Token::kASSIGN); | 257 CheckKind(tokens, 1, Token::kASSIGN); |
| 277 CheckKind(tokens, 2, Token::kSTRING); | 258 CheckKind(tokens, 2, Token::kSTRING); |
| 278 CheckKind(tokens, 3, Token::kSEMICOLON); | 259 CheckKind(tokens, 3, Token::kSEMICOLON); |
| 279 CheckKind(tokens, 4, Token::kEOS); | 260 CheckKind(tokens, 4, Token::kEOS); |
| 280 EXPECT_EQ(0, (tokens)[2].literal->Length()); | 261 EXPECT_EQ(0, (tokens)[2].literal->Length()); |
| 281 } | 262 } |
| 282 | 263 |
| 283 | |
| 284 static void NumberLiteral() { | 264 static void NumberLiteral() { |
| 285 const GrowableTokenStream& tokens = Scan("5 0x5d 0.3 0.33 1E+12 .42 +5"); | 265 const GrowableTokenStream& tokens = Scan("5 0x5d 0.3 0.33 1E+12 .42 +5"); |
| 286 | 266 |
| 287 CheckKind(tokens, 0, Token::kINTEGER); | 267 CheckKind(tokens, 0, Token::kINTEGER); |
| 288 CheckKind(tokens, 1, Token::kINTEGER); | 268 CheckKind(tokens, 1, Token::kINTEGER); |
| 289 CheckKind(tokens, 2, Token::kDOUBLE); | 269 CheckKind(tokens, 2, Token::kDOUBLE); |
| 290 CheckKind(tokens, 3, Token::kDOUBLE); | 270 CheckKind(tokens, 3, Token::kDOUBLE); |
| 291 CheckKind(tokens, 4, Token::kDOUBLE); | 271 CheckKind(tokens, 4, Token::kDOUBLE); |
| 292 CheckKind(tokens, 5, Token::kDOUBLE); | 272 CheckKind(tokens, 5, Token::kDOUBLE); |
| 293 CheckKind(tokens, 6, Token::kADD); | 273 CheckKind(tokens, 6, Token::kADD); |
| 294 CheckKind(tokens, 7, Token::kINTEGER); | 274 CheckKind(tokens, 7, Token::kINTEGER); |
| 295 CheckKind(tokens, 8, Token::kEOS); | 275 CheckKind(tokens, 8, Token::kEOS); |
| 296 } | 276 } |
| 297 | 277 |
| 298 | |
| 299 static void ScanLargeText() { | 278 static void ScanLargeText() { |
| 300 const char* dart_source = | 279 const char* dart_source = |
| 301 "// This source is not meant to be valid Dart code. The text is used to" | 280 "// This source is not meant to be valid Dart code. The text is used to" |
| 302 "// test the Dart scanner." | 281 "// test the Dart scanner." |
| 303 "" | 282 "" |
| 304 "// Cartesian point implementation." | 283 "// Cartesian point implementation." |
| 305 "class Point {" | 284 "class Point {" |
| 306 "" | 285 "" |
| 307 " // Constructor" | 286 " // Constructor" |
| 308 " Point(Number x, Number y) : x(x), y(y) { }" | 287 " Point(Number x, Number y) : x(x), y(y) { }" |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 352 " for (int i = 0; i < this.length; i++) {" | 331 " for (int i = 0; i < this.length; i++) {" |
| 353 " f(this[i]);" | 332 " f(this[i]);" |
| 354 " }" | 333 " }" |
| 355 "}" | 334 "}" |
| 356 "" | 335 "" |
| 357 "" | 336 "" |
| 358 "j!==!iffy // means j !== !iffy"; | 337 "j!==!iffy // means j !== !iffy"; |
| 359 Scan(dart_source); | 338 Scan(dart_source); |
| 360 } | 339 } |
| 361 | 340 |
| 362 | |
| 363 void InvalidText() { | 341 void InvalidText() { |
| 364 const GrowableTokenStream& tokens = Scan("\\"); | 342 const GrowableTokenStream& tokens = Scan("\\"); |
| 365 | 343 |
| 366 EXPECT_EQ(2, tokens.length()); | 344 EXPECT_EQ(2, tokens.length()); |
| 367 CheckKind(tokens, 0, Token::kERROR); | 345 CheckKind(tokens, 0, Token::kERROR); |
| 368 CheckKind(tokens, 1, Token::kEOS); | 346 CheckKind(tokens, 1, Token::kEOS); |
| 369 } | 347 } |
| 370 | 348 |
| 371 | |
| 372 void NewlinesTest() { | 349 void NewlinesTest() { |
| 373 const char* source = | 350 const char* source = |
| 374 "var es = /* a\n" | 351 "var es = /* a\n" |
| 375 " b\n" | 352 " b\n" |
| 376 " */ \"\"\"\n" | 353 " */ \"\"\"\n" |
| 377 "c\n" | 354 "c\n" |
| 378 "d\n" | 355 "d\n" |
| 379 "\"\"\";"; | 356 "\"\"\";"; |
| 380 | 357 |
| 381 const GrowableTokenStream& tokens = Scan(source); | 358 const GrowableTokenStream& tokens = Scan(source); |
| (...skipping 12 matching lines...) Expand all Loading... |
| 394 CheckKind(tokens, 10, Token::kEOS); | 371 CheckKind(tokens, 10, Token::kEOS); |
| 395 | 372 |
| 396 EXPECT_EQ(4, (tokens)[5].literal->Length()); | 373 EXPECT_EQ(4, (tokens)[5].literal->Length()); |
| 397 const char* litchars = (tokens)[5].literal->ToCString(); | 374 const char* litchars = (tokens)[5].literal->ToCString(); |
| 398 EXPECT_EQ('c', litchars[0]); // First newline is dropped. | 375 EXPECT_EQ('c', litchars[0]); // First newline is dropped. |
| 399 EXPECT_EQ('\n', litchars[1]); | 376 EXPECT_EQ('\n', litchars[1]); |
| 400 EXPECT_EQ('d', litchars[2]); | 377 EXPECT_EQ('d', litchars[2]); |
| 401 EXPECT_EQ('\n', litchars[3]); | 378 EXPECT_EQ('\n', litchars[3]); |
| 402 } | 379 } |
| 403 | 380 |
| 404 | |
| 405 TEST_CASE(Scanner_Test) { | 381 TEST_CASE(Scanner_Test) { |
| 406 ScanLargeText(); | 382 ScanLargeText(); |
| 407 | 383 |
| 408 BoringTest(); | 384 BoringTest(); |
| 409 CommentTest(); | 385 CommentTest(); |
| 410 GreedIsGood(); | 386 GreedIsGood(); |
| 411 StringEscapes(); | 387 StringEscapes(); |
| 412 InvalidStringEscapes(); | 388 InvalidStringEscapes(); |
| 413 RawString(); | 389 RawString(); |
| 414 MultilineString(); | 390 MultilineString(); |
| 415 EmptyString(); | 391 EmptyString(); |
| 416 EmptyMultilineString(); | 392 EmptyMultilineString(); |
| 417 NumberLiteral(); | 393 NumberLiteral(); |
| 418 InvalidText(); | 394 InvalidText(); |
| 419 NewlinesTest(); | 395 NewlinesTest(); |
| 420 } | 396 } |
| 421 | 397 |
| 422 } // namespace dart | 398 } // namespace dart |
| OLD | NEW |