| OLD | NEW |
| 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library engine.scanner; | 5 library engine.scanner; |
| 6 | 6 |
| 7 import 'dart:collection'; | 7 import 'dart:collection'; |
| 8 | 8 |
| 9 import 'error.dart'; | 9 import 'error.dart'; |
| 10 import 'instrumentation.dart'; | |
| 11 import 'java_engine.dart'; | 10 import 'java_engine.dart'; |
| 12 import 'source.dart'; | 11 import 'source.dart'; |
| 13 | 12 |
| 14 /** | 13 /** |
| 15 * A `BeginToken` is the opening half of a grouping pair of tokens. This is used | 14 * A `BeginToken` is the opening half of a grouping pair of tokens. This is used |
| 16 * for curly brackets ('{'), parentheses ('('), and square brackets ('['). | 15 * for curly brackets ('{'), parentheses ('('), and square brackets ('['). |
| 17 */ | 16 */ |
| 18 class BeginToken extends Token { | 17 class BeginToken extends Token { |
| 19 /** | 18 /** |
| 20 * The token that corresponds to this token. | 19 * The token that corresponds to this token. |
| (...skipping 945 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 966 _lineStarts.add(1); | 965 _lineStarts.add(1); |
| 967 } | 966 } |
| 968 _lineStarts.add(offset - column + 1); | 967 _lineStarts.add(offset - column + 1); |
| 969 } | 968 } |
| 970 | 969 |
| 971 /** | 970 /** |
| 972 * Scan the source code to produce a list of tokens representing the source, | 971 * Scan the source code to produce a list of tokens representing the source, |
| 973 * and return the first token in the list of tokens that were produced. | 972 * and return the first token in the list of tokens that were produced. |
| 974 */ | 973 */ |
| 975 Token tokenize() { | 974 Token tokenize() { |
| 976 InstrumentationBuilder instrumentation = | 975 int next = _reader.advance(); |
| 977 Instrumentation.builder2("dart.engine.AbstractScanner.tokenize"); | 976 while (next != -1) { |
| 978 int tokenCounter = 0; | 977 next = bigSwitch(next); |
| 979 try { | |
| 980 int next = _reader.advance(); | |
| 981 while (next != -1) { | |
| 982 tokenCounter++; | |
| 983 next = bigSwitch(next); | |
| 984 } | |
| 985 _appendEofToken(); | |
| 986 instrumentation.metric2("tokensCount", tokenCounter); | |
| 987 return firstToken; | |
| 988 } finally { | |
| 989 instrumentation.log2(2); | |
| 990 //Log if over 1ms | |
| 991 } | 978 } |
| 979 _appendEofToken(); |
| 980 return firstToken; |
| 992 } | 981 } |
| 993 | 982 |
| 994 void _appendBeginToken(TokenType type) { | 983 void _appendBeginToken(TokenType type) { |
| 995 BeginToken token; | 984 BeginToken token; |
| 996 if (_firstComment == null) { | 985 if (_firstComment == null) { |
| 997 token = new BeginToken(type, _tokenStart); | 986 token = new BeginToken(type, _tokenStart); |
| 998 } else { | 987 } else { |
| 999 token = new BeginTokenWithComment(type, _tokenStart, _firstComment); | 988 token = new BeginTokenWithComment(type, _tokenStart, _firstComment); |
| 1000 _firstComment = null; | 989 _firstComment = null; |
| 1001 _lastComment = null; | 990 _lastComment = null; |
| (...skipping 1646 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2648 CommentToken get precedingComments => _precedingComment; | 2637 CommentToken get precedingComments => _precedingComment; |
| 2649 | 2638 |
| 2650 void set precedingComments(CommentToken comment) { | 2639 void set precedingComments(CommentToken comment) { |
| 2651 _precedingComment = comment; | 2640 _precedingComment = comment; |
| 2652 _setCommentParent(_precedingComment); | 2641 _setCommentParent(_precedingComment); |
| 2653 } | 2642 } |
| 2654 | 2643 |
| 2655 @override | 2644 @override |
| 2656 Token copy() => new TokenWithComment(type, offset, precedingComments); | 2645 Token copy() => new TokenWithComment(type, offset, precedingComments); |
| 2657 } | 2646 } |
| OLD | NEW |