Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(374)

Side by Side Diff: pkg/front_end/lib/src/fasta/parser/token_stream_rewriter.dart

Issue 2899043006: cleanup fasta token classes (Closed)
Patch Set: Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 import 'package:front_end/src/fasta/errors.dart'; 5 import 'package:front_end/src/fasta/errors.dart';
6 import 'package:front_end/src/fasta/scanner/token.dart'; 6 import 'package:front_end/src/scanner/token.dart' show BeginToken, Token;
7 import 'package:front_end/src/scanner/token.dart' show Token;
8 7
9 /// Provides the capability of inserting tokens into a token stream by rewriting 8 /// Provides the capability of inserting tokens into a token stream by rewriting
10 /// the previous token to point to the inserted token. 9 /// the previous token to point to the inserted token.
11 /// 10 ///
12 /// This class has been designed to take advantage of "previousToken" pointers 11 /// This class has been designed to take advantage of "previousToken" pointers
13 /// when they are present, but not to depend on them. When they are not 12 /// when they are present, but not to depend on them. When they are not
14 /// present, it uses heuristics to try to find the find the previous token as 13 /// present, it uses heuristics to try to find the find the previous token as
15 /// quickly as possible by walking through tokens starting at the start of the 14 /// quickly as possible by walking through tokens starting at the start of the
16 /// file. 15 /// file.
17 class TokenStreamRewriter { 16 class TokenStreamRewriter {
18 /// Synthetic token whose "next" pointer points to the first token in the 17 /// Synthetic token whose "next" pointer points to the first token in the
19 /// stream. 18 /// stream.
20 final Token _head; 19 final Token _head;
21 20
22 /// The token whose "next" pointer was updated in the last call to 21 /// The token whose "next" pointer was updated in the last call to
23 /// [insertTokenBefore]. This can often be used as a starting point to find 22 /// [insertTokenBefore]. This can often be used as a starting point to find
24 /// the a future insertion point quickly. 23 /// the a future insertion point quickly.
25 Token _lastPreviousToken; 24 Token _lastPreviousToken;
26 25
27 /// Creates a [TokenStreamRewriter] which is prepared to rewrite the token 26 /// Creates a [TokenStreamRewriter] which is prepared to rewrite the token
28 /// stream whose first token is [firstToken]. 27 /// stream whose first token is [firstToken].
29 TokenStreamRewriter(Token firstToken) 28 TokenStreamRewriter(Token firstToken)
30 : _head = 29 : _head = firstToken.previous ?? (new Token.eof(-1)..next = firstToken);
31 firstToken.previous ?? (new SymbolToken.eof(-1)..next = firstToken);
32 30
33 /// Gets the first token in the stream (which may not be the same token that 31 /// Gets the first token in the stream (which may not be the same token that
34 /// was passed to the constructor, if something was inserted before it). 32 /// was passed to the constructor, if something was inserted before it).
35 Token get firstToken => _head.next; 33 Token get firstToken => _head.next;
36 34
37 /// Inserts [newToken] into the token stream just before [insertionPoint], and 35 /// Inserts [newToken] into the token stream just before [insertionPoint], and
38 /// fixes up all "next" and "previous" pointers. 36 /// fixes up all "next" and "previous" pointers.
39 /// 37 ///
40 /// Caller is required to ensure that [insertionPoint] is actually present in 38 /// Caller is required to ensure that [insertionPoint] is actually present in
41 /// the token stream. 39 /// the token stream.
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
82 80
83 /// Searches for the token that immediately precedes [target], using [pos] as 81 /// Searches for the token that immediately precedes [target], using [pos] as
84 /// a starting point. 82 /// a starting point.
85 /// 83 ///
86 /// Uses heuristics to skip matching `{}`, `[]`, `()`, and `<>` if possible. 84 /// Uses heuristics to skip matching `{}`, `[]`, `()`, and `<>` if possible.
87 /// 85 ///
88 /// If no such token is found, returns `null`. 86 /// If no such token is found, returns `null`.
89 Token _scanForPreviousToken(Token target, Token pos) { 87 Token _scanForPreviousToken(Token target, Token pos) {
90 while (!identical(pos.next, target)) { 88 while (!identical(pos.next, target)) {
91 Token nextPos; 89 Token nextPos;
92 if (pos is BeginGroupToken && 90 if (pos is BeginToken &&
93 pos.endGroup != null && 91 pos.endGroup != null &&
94 pos.endGroup.charOffset < target.charOffset) { 92 pos.endGroup.charOffset < target.charOffset) {
95 nextPos = pos.endGroup; 93 nextPos = pos.endGroup;
96 } else { 94 } else {
97 nextPos = pos.next; 95 nextPos = pos.next;
98 if (nextPos == null || nextPos.charOffset > target.charOffset) { 96 if (nextPos == null || nextPos.charOffset > target.charOffset) {
99 return null; 97 return null;
100 } 98 }
101 } 99 }
102 pos = nextPos; 100 pos = nextPos;
103 } 101 }
104 return pos; 102 return pos;
105 } 103 }
106 } 104 }
OLDNEW
« no previous file with comments | « pkg/front_end/lib/src/fasta/parser/parser.dart ('k') | pkg/front_end/lib/src/fasta/scanner.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698