| OLD | NEW |
| 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #ifndef TOOLS_GN_TOKENIZER_H_ | 5 #ifndef TOOLS_GN_TOKENIZER_H_ |
| 6 #define TOOLS_GN_TOKENIZER_H_ | 6 #define TOOLS_GN_TOKENIZER_H_ |
| 7 | 7 |
| 8 #include <vector> | 8 #include <vector> |
| 9 | 9 |
| 10 #include "base/basictypes.h" | 10 #include "base/basictypes.h" |
| (...skipping 15 matching lines...) Expand all Loading... |
| 26 // | 26 // |
| 27 // This is a helper function for error output so that the tokenizer's | 27 // This is a helper function for error output so that the tokenizer's |
| 28 // notion of lines can be used elsewhere. | 28 // notion of lines can be used elsewhere. |
| 29 static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); | 29 static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); |
| 30 | 30 |
| 31 // Returns true if the given offset of the string piece counts as a newline. | 31 // Returns true if the given offset of the string piece counts as a newline. |
| 32 // The offset must be in the buffer. | 32 // The offset must be in the buffer. |
| 33 static bool IsNewline(const base::StringPiece& buffer, size_t offset); | 33 static bool IsNewline(const base::StringPiece& buffer, size_t offset); |
| 34 | 34 |
| 35 static bool IsIdentifierFirstChar(char c) { | 35 static bool IsIdentifierFirstChar(char c) { |
| 36 return IsAsciiAlpha(c) || c == '_'; | 36 return base::IsAsciiAlpha(c) || c == '_'; |
| 37 } | 37 } |
| 38 | 38 |
| 39 static bool IsIdentifierContinuingChar(char c) { | 39 static bool IsIdentifierContinuingChar(char c) { |
| 40 // Also allow digits after the first char. | 40 // Also allow digits after the first char. |
| 41 return IsIdentifierFirstChar(c) || IsAsciiDigit(c); | 41 return IsIdentifierFirstChar(c) || base::IsAsciiDigit(c); |
| 42 } | 42 } |
| 43 | 43 |
| 44 private: | 44 private: |
| 45 // InputFile must outlive the tokenizer and all generated tokens. | 45 // InputFile must outlive the tokenizer and all generated tokens. |
| 46 Tokenizer(const InputFile* input_file, Err* err); | 46 Tokenizer(const InputFile* input_file, Err* err); |
| 47 ~Tokenizer(); | 47 ~Tokenizer(); |
| 48 | 48 |
| 49 std::vector<Token> Run(); | 49 std::vector<Token> Run(); |
| 50 | 50 |
| 51 void AdvanceToNextToken(); | 51 void AdvanceToNextToken(); |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 85 Err* err_; | 85 Err* err_; |
| 86 size_t cur_; // Byte offset into input buffer. | 86 size_t cur_; // Byte offset into input buffer. |
| 87 | 87 |
| 88 int line_number_; | 88 int line_number_; |
| 89 int char_in_line_; | 89 int char_in_line_; |
| 90 | 90 |
| 91 DISALLOW_COPY_AND_ASSIGN(Tokenizer); | 91 DISALLOW_COPY_AND_ASSIGN(Tokenizer); |
| 92 }; | 92 }; |
| 93 | 93 |
| 94 #endif // TOOLS_GN_TOKENIZER_H_ | 94 #endif // TOOLS_GN_TOKENIZER_H_ |
| OLD | NEW |