Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(332)

Side by Side Diff: pkg/front_end/lib/src/fasta/parser/token_stream_rewriter.dart

Issue 2872433005: more work aligning fasta.Token with analyzer.Token (Closed)
Patch Set: Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 import 'package:front_end/src/fasta/errors.dart'; 5 import 'package:front_end/src/fasta/errors.dart';
6 import 'package:front_end/src/fasta/scanner/token.dart'; 6 import 'package:front_end/src/fasta/scanner/token.dart';
7 7
8 /// Provides the capability of inserting tokens into a token stream by rewriting 8 /// Provides the capability of inserting tokens into a token stream by rewriting
9 /// the previous token to point to the inserted token. 9 /// the previous token to point to the inserted token.
10 /// 10 ///
11 /// This class has been designed to take advantage of "previousToken" pointers 11 /// This class has been designed to take advantage of "previousToken" pointers
12 /// when they are present, but not to depend on them. When they are not 12 /// when they are present, but not to depend on them. When they are not
13 /// present, it uses heuristics to try to find the find the previous token as 13 /// present, it uses heuristics to try to find the find the previous token as
14 /// quickly as possible by walking through tokens starting at the start of the 14 /// quickly as possible by walking through tokens starting at the start of the
15 /// file. 15 /// file.
16 class TokenStreamRewriter { 16 class TokenStreamRewriter {
17 /// Synthetic token whose "next" pointer points to the first token in the 17 /// Synthetic token whose "next" pointer points to the first token in the
18 /// stream. 18 /// stream.
19 final Token _head; 19 final Token _head;
20 20
21 /// The token whose "next" pointer was updated in the last call to 21 /// The token whose "next" pointer was updated in the last call to
22 /// [insertTokenBefore]. This can often be used as a starting point to find 22 /// [insertTokenBefore]. This can often be used as a starting point to find
23 /// the a future insertion point quickly. 23 /// the a future insertion point quickly.
24 Token _lastPreviousToken; 24 Token _lastPreviousToken;
25 25
26 /// Creates a [TokenStreamRewriter] which is prepared to rewrite the token 26 /// Creates a [TokenStreamRewriter] which is prepared to rewrite the token
27 /// stream whose first token is [firstToken]. 27 /// stream whose first token is [firstToken].
28 TokenStreamRewriter(Token firstToken) 28 TokenStreamRewriter(Token firstToken)
29 : _head = firstToken.previousToken ?? 29 : _head =
30 (new SymbolToken.eof(-1)..next = firstToken); 30 firstToken.previous ?? (new SymbolToken.eof(-1)..next = firstToken);
31 31
32 /// Gets the first token in the stream (which may not be the same token that 32 /// Gets the first token in the stream (which may not be the same token that
33 /// was passed to the constructor, if something was inserted before it). 33 /// was passed to the constructor, if something was inserted before it).
34 Token get firstToken => _head.next; 34 Token get firstToken => _head.next;
35 35
36 /// Inserts [newToken] into the token stream just before [insertionPoint], and 36 /// Inserts [newToken] into the token stream just before [insertionPoint], and
37 /// fixes up all "next" and "previous" pointers. 37 /// fixes up all "next" and "previous" pointers.
38 /// 38 ///
39 /// Caller is required to ensure that [insertionPoint] is actually present in 39 /// Caller is required to ensure that [insertionPoint] is actually present in
40 /// the token stream. 40 /// the token stream.
41 void insertTokenBefore(Token newToken, Token insertionPoint) { 41 void insertTokenBefore(Token newToken, Token insertionPoint) {
42 Token previous = _findPreviousToken(insertionPoint); 42 Token previous = _findPreviousToken(insertionPoint);
43 _lastPreviousToken = previous; 43 _lastPreviousToken = previous;
44 newToken.next = insertionPoint; 44 newToken.next = insertionPoint;
45 previous.next = newToken; 45 previous.next = newToken;
46 { 46 {
47 // Note: even though previousToken is deprecated, we need to hook it up in 47 // Note: even though previousToken is deprecated, we need to hook it up in
48 // case any uses of it remain. Once previousToken is removed it should be 48 // case any uses of it remain. Once previousToken is removed it should be
49 // safe to remove this block of code. 49 // safe to remove this block of code.
50 insertionPoint.previousToken = newToken; 50 insertionPoint.previous = newToken;
51 newToken.previousToken = previous; 51 newToken.previous = previous;
52 } 52 }
53 } 53 }
54 54
55 /// Finds the token that immediately precedes [target]. 55 /// Finds the token that immediately precedes [target].
56 Token _findPreviousToken(Token target) { 56 Token _findPreviousToken(Token target) {
57 // First see if the target has a previous token pointer. If it does, then 57 // First see if the target has a previous token pointer. If it does, then
58 // we can find the previous token with no extra effort. Note: it's ok that 58 // we can find the previous token with no extra effort. Note: it's ok that
59 // we're accessing the deprecated member previousToken here, because we have 59 // we're accessing the deprecated member previousToken here, because we have
60 // a fallback if it is not available. Once previousToken is removed, we can 60 // a fallback if it is not available. Once previousToken is removed, we can
61 // remove the "if" test below, and always use the fallback code. 61 // remove the "if" test below, and always use the fallback code.
62 if (target.previousToken != null) { 62 if (target.previous != null) {
63 return target.previousToken; 63 return target.previous;
64 } 64 }
65 65
66 // Look for the previous token by scanning forward from [lastPreviousToken], 66 // Look for the previous token by scanning forward from [lastPreviousToken],
67 // if it makes sense to do so. 67 // if it makes sense to do so.
68 if (_lastPreviousToken != null && 68 if (_lastPreviousToken != null &&
69 target.charOffset >= _lastPreviousToken.charOffset) { 69 target.charOffset >= _lastPreviousToken.charOffset) {
70 Token previous = _scanForPreviousToken(target, _lastPreviousToken); 70 Token previous = _scanForPreviousToken(target, _lastPreviousToken);
71 if (previous != null) return previous; 71 if (previous != null) return previous;
72 } 72 }
73 73
(...skipping 22 matching lines...) Expand all
96 nextPos = pos.next; 96 nextPos = pos.next;
97 if (nextPos == null || nextPos.charOffset > target.charOffset) { 97 if (nextPos == null || nextPos.charOffset > target.charOffset) {
98 return null; 98 return null;
99 } 99 }
100 } 100 }
101 pos = nextPos; 101 pos = nextPos;
102 } 102 }
103 return pos; 103 return pos;
104 } 104 }
105 } 105 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698