OLD | NEW |
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #ifndef TOOLS_GN_TOKENIZER_H_ | 5 #ifndef TOOLS_GN_TOKENIZER_H_ |
6 #define TOOLS_GN_TOKENIZER_H_ | 6 #define TOOLS_GN_TOKENIZER_H_ |
7 | 7 |
8 #include <vector> | 8 #include <vector> |
9 | 9 |
10 #include "base/basictypes.h" | 10 #include "base/basictypes.h" |
(...skipping 15 matching lines...) Expand all Loading... |
26 // | 26 // |
27 // This is a helper function for error output so that the tokenizer's | 27 // This is a helper function for error output so that the tokenizer's |
28 // notion of lines can be used elsewhere. | 28 // notion of lines can be used elsewhere. |
29 static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); | 29 static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); |
30 | 30 |
31 // Returns true if the given offset of the string piece counts as a newline. | 31 // Returns true if the given offset of the string piece counts as a newline. |
32 // The offset must be in the buffer. | 32 // The offset must be in the buffer. |
33 static bool IsNewline(const base::StringPiece& buffer, size_t offset); | 33 static bool IsNewline(const base::StringPiece& buffer, size_t offset); |
34 | 34 |
35 static bool IsIdentifierFirstChar(char c) { | 35 static bool IsIdentifierFirstChar(char c) { |
36 return IsAsciiAlpha(c) || c == '_'; | 36 return base::IsAsciiAlpha(c) || c == '_'; |
37 } | 37 } |
38 | 38 |
39 static bool IsIdentifierContinuingChar(char c) { | 39 static bool IsIdentifierContinuingChar(char c) { |
40 // Also allow digits after the first char. | 40 // Also allow digits after the first char. |
41 return IsIdentifierFirstChar(c) || IsAsciiDigit(c); | 41 return IsIdentifierFirstChar(c) || base::IsAsciiDigit(c); |
42 } | 42 } |
43 | 43 |
44 private: | 44 private: |
45 // InputFile must outlive the tokenizer and all generated tokens. | 45 // InputFile must outlive the tokenizer and all generated tokens. |
46 Tokenizer(const InputFile* input_file, Err* err); | 46 Tokenizer(const InputFile* input_file, Err* err); |
47 ~Tokenizer(); | 47 ~Tokenizer(); |
48 | 48 |
49 std::vector<Token> Run(); | 49 std::vector<Token> Run(); |
50 | 50 |
51 void AdvanceToNextToken(); | 51 void AdvanceToNextToken(); |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
85 Err* err_; | 85 Err* err_; |
86 size_t cur_; // Byte offset into input buffer. | 86 size_t cur_; // Byte offset into input buffer. |
87 | 87 |
88 int line_number_; | 88 int line_number_; |
89 int char_in_line_; | 89 int char_in_line_; |
90 | 90 |
91 DISALLOW_COPY_AND_ASSIGN(Tokenizer); | 91 DISALLOW_COPY_AND_ASSIGN(Tokenizer); |
92 }; | 92 }; |
93 | 93 |
94 #endif // TOOLS_GN_TOKENIZER_H_ | 94 #endif // TOOLS_GN_TOKENIZER_H_ |
OLD | NEW |