| OLD | NEW | 
|---|
| 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be | 
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. | 
| 4 | 4 | 
| 5 #ifndef TOOLS_GN_TOKENIZER_H_ | 5 #ifndef TOOLS_GN_TOKENIZER_H_ | 
| 6 #define TOOLS_GN_TOKENIZER_H_ | 6 #define TOOLS_GN_TOKENIZER_H_ | 
| 7 | 7 | 
| 8 #include <vector> | 8 #include <vector> | 
| 9 | 9 | 
| 10 #include "base/basictypes.h" | 10 #include "base/basictypes.h" | 
| (...skipping 15 matching lines...) Expand all  Loading... | 
| 26   // | 26   // | 
| 27   // This is a helper function for error output so that the tokenizer's | 27   // This is a helper function for error output so that the tokenizer's | 
| 28   // notion of lines can be used elsewhere. | 28   // notion of lines can be used elsewhere. | 
| 29   static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); | 29   static size_t ByteOffsetOfNthLine(const base::StringPiece& buf, int n); | 
| 30 | 30 | 
| 31   // Returns true if the given offset of the string piece counts as a newline. | 31   // Returns true if the given offset of the string piece counts as a newline. | 
| 32   // The offset must be in the buffer. | 32   // The offset must be in the buffer. | 
| 33   static bool IsNewline(const base::StringPiece& buffer, size_t offset); | 33   static bool IsNewline(const base::StringPiece& buffer, size_t offset); | 
| 34 | 34 | 
| 35   static bool IsIdentifierFirstChar(char c) { | 35   static bool IsIdentifierFirstChar(char c) { | 
| 36     return IsAsciiAlpha(c) || c == '_'; | 36     return base::IsAsciiAlpha(c) || c == '_'; | 
| 37   } | 37   } | 
| 38 | 38 | 
| 39   static bool IsIdentifierContinuingChar(char c) { | 39   static bool IsIdentifierContinuingChar(char c) { | 
| 40     // Also allow digits after the first char. | 40     // Also allow digits after the first char. | 
| 41     return IsIdentifierFirstChar(c) || IsAsciiDigit(c); | 41     return IsIdentifierFirstChar(c) || base::IsAsciiDigit(c); | 
| 42   } | 42   } | 
| 43 | 43 | 
| 44  private: | 44  private: | 
| 45   // InputFile must outlive the tokenizer and all generated tokens. | 45   // InputFile must outlive the tokenizer and all generated tokens. | 
| 46   Tokenizer(const InputFile* input_file, Err* err); | 46   Tokenizer(const InputFile* input_file, Err* err); | 
| 47   ~Tokenizer(); | 47   ~Tokenizer(); | 
| 48 | 48 | 
| 49   std::vector<Token> Run(); | 49   std::vector<Token> Run(); | 
| 50 | 50 | 
| 51   void AdvanceToNextToken(); | 51   void AdvanceToNextToken(); | 
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 85   Err* err_; | 85   Err* err_; | 
| 86   size_t cur_;  // Byte offset into input buffer. | 86   size_t cur_;  // Byte offset into input buffer. | 
| 87 | 87 | 
| 88   int line_number_; | 88   int line_number_; | 
| 89   int char_in_line_; | 89   int char_in_line_; | 
| 90 | 90 | 
| 91   DISALLOW_COPY_AND_ASSIGN(Tokenizer); | 91   DISALLOW_COPY_AND_ASSIGN(Tokenizer); | 
| 92 }; | 92 }; | 
| 93 | 93 | 
| 94 #endif  // TOOLS_GN_TOKENIZER_H_ | 94 #endif  // TOOLS_GN_TOKENIZER_H_ | 
| OLD | NEW | 
|---|