OLD | NEW |
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #ifndef TOOLS_GN_TOKENIZER_H_ | 5 #ifndef TOOLS_GN_TOKENIZER_H_ |
6 #define TOOLS_GN_TOKENIZER_H_ | 6 #define TOOLS_GN_TOKENIZER_H_ |
7 | 7 |
8 #include <vector> | 8 #include <vector> |
9 | 9 |
10 #include "base/basictypes.h" | 10 #include "base/basictypes.h" |
(...skipping 15 matching lines...) Expand all Loading... |
26 // | 26 // |
27 // This is a helper function for error output so that the tokenizer's | 27 // This is a helper function for error output so that the tokenizer's |
28 // notion of lines can be used elsewhere. | 28 // notion of lines can be used elsewhere. |
29 static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); | 29 static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); |
30 | 30 |
31 // Returns true if the given offset of the string piece counts as a newline. | 31 // Returns true if the given offset of the string piece counts as a newline. |
32 // The offset must be in the buffer. | 32 // The offset must be in the buffer. |
33 static bool IsNewline(const base::StringPiece& buffer, size_t offset); | 33 static bool IsNewline(const base::StringPiece& buffer, size_t offset); |
34 | 34 |
35 static bool IsIdentifierFirstChar(char c) { | 35 static bool IsIdentifierFirstChar(char c) { |
36 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; | 36 return IsAsciiAlpha(c) || c == '_'; |
37 } | 37 } |
38 | 38 |
39 static bool IsIdentifierContinuingChar(char c) { | 39 static bool IsIdentifierContinuingChar(char c) { |
40 // Also allow digits after the first char. | 40 // Also allow digits after the first char. |
41 return IsIdentifierFirstChar(c) || IsAsciiDigit(c); | 41 return IsIdentifierFirstChar(c) || IsAsciiDigit(c); |
42 } | 42 } |
43 | 43 |
44 private: | 44 private: |
45 // InputFile must outlive the tokenizer and all generated tokens. | 45 // InputFile must outlive the tokenizer and all generated tokens. |
46 explicit Tokenizer(const InputFile* input_file, Err* err); | 46 explicit Tokenizer(const InputFile* input_file, Err* err); |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
80 Err* err_; | 80 Err* err_; |
81 size_t cur_; // Byte offset into input buffer. | 81 size_t cur_; // Byte offset into input buffer. |
82 | 82 |
83 int line_number_; | 83 int line_number_; |
84 int char_in_line_; | 84 int char_in_line_; |
85 | 85 |
86 DISALLOW_COPY_AND_ASSIGN(Tokenizer); | 86 DISALLOW_COPY_AND_ASSIGN(Tokenizer); |
87 }; | 87 }; |
88 | 88 |
89 #endif // TOOLS_GN_TOKENIZER_H_ | 89 #endif // TOOLS_GN_TOKENIZER_H_ |
OLD | NEW |