OLD | NEW |
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #ifndef TOOLS_GN_TOKENIZER_H_ | 5 #ifndef TOOLS_GN_TOKENIZER_H_ |
6 #define TOOLS_GN_TOKENIZER_H_ | 6 #define TOOLS_GN_TOKENIZER_H_ |
7 | 7 |
8 #include <vector> | 8 #include <vector> |
9 | 9 |
10 #include "base/basictypes.h" | 10 #include "base/basictypes.h" |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
45 // InputFile must outlive the tokenizer and all generated tokens. | 45 // InputFile must outlive the tokenizer and all generated tokens. |
46 explicit Tokenizer(const InputFile* input_file, Err* err); | 46 explicit Tokenizer(const InputFile* input_file, Err* err); |
47 ~Tokenizer(); | 47 ~Tokenizer(); |
48 | 48 |
49 std::vector<Token> Run(); | 49 std::vector<Token> Run(); |
50 | 50 |
51 void AdvanceToNextToken(); | 51 void AdvanceToNextToken(); |
52 Token::Type ClassifyCurrent() const; | 52 Token::Type ClassifyCurrent() const; |
53 void AdvanceToEndOfToken(const Location& location, Token::Type type); | 53 void AdvanceToEndOfToken(const Location& location, Token::Type type); |
54 | 54 |
| 55 // Whether from this location back to the beginning of the line is only |
| 56 // whitespace. |location| should be the first character of the token to be |
| 57 // checked. |
| 58 bool AtStartOfLine(size_t location) const; |
| 59 |
55 bool IsCurrentWhitespace() const; | 60 bool IsCurrentWhitespace() const; |
56 bool IsCurrentNewline() const; | 61 bool IsCurrentNewline() const; |
57 bool IsCurrentStringTerminator(char quote_char) const; | 62 bool IsCurrentStringTerminator(char quote_char) const; |
58 | 63 |
59 bool CanIncrement() const { return cur_ < input_.size(); } | 64 bool CanIncrement() const { return cur_ < input_.size(); } |
60 | 65 |
61 // Increments the current location by one. | 66 // Increments the current location by one. |
62 void Advance(); | 67 void Advance(); |
63 | 68 |
64 // Returns the current character in the file as a location. | 69 // Returns the current character in the file as a location. |
(...skipping 15 matching lines...) Expand all Loading... |
80 Err* err_; | 85 Err* err_; |
81 size_t cur_; // Byte offset into input buffer. | 86 size_t cur_; // Byte offset into input buffer. |
82 | 87 |
83 int line_number_; | 88 int line_number_; |
84 int char_in_line_; | 89 int char_in_line_; |
85 | 90 |
86 DISALLOW_COPY_AND_ASSIGN(Tokenizer); | 91 DISALLOW_COPY_AND_ASSIGN(Tokenizer); |
87 }; | 92 }; |
88 | 93 |
89 #endif // TOOLS_GN_TOKENIZER_H_ | 94 #endif // TOOLS_GN_TOKENIZER_H_ |
OLD | NEW |