Index: third_party/closure_linter/closure_linter/common/tokens.py |
diff --git a/third_party/closure_linter/closure_linter/common/tokens.py b/third_party/closure_linter/closure_linter/common/tokens.py |
deleted file mode 100755 |
index 4703998752b03670ba5f3718483fb97fbaf77203..0000000000000000000000000000000000000000 |
--- a/third_party/closure_linter/closure_linter/common/tokens.py |
+++ /dev/null |
@@ -1,145 +0,0 @@ |
-#!/usr/bin/env python |
-# |
-# Copyright 2008 The Closure Linter Authors. All Rights Reserved. |
-# |
-# Licensed under the Apache License, Version 2.0 (the "License"); |
-# you may not use this file except in compliance with the License. |
-# You may obtain a copy of the License at |
-# |
-# http://www.apache.org/licenses/LICENSE-2.0 |
-# |
-# Unless required by applicable law or agreed to in writing, software |
-# distributed under the License is distributed on an "AS-IS" BASIS, |
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
-# See the License for the specific language governing permissions and |
-# limitations under the License. |
- |
-"""Classes to represent tokens and positions within them.""" |
- |
-__author__ = ('robbyw@google.com (Robert Walker)', |
- 'ajp@google.com (Andy Perelson)') |
- |
- |
-class TokenType(object): |
- """Token types common to all languages.""" |
- NORMAL = 'normal' |
- WHITESPACE = 'whitespace' |
- BLANK_LINE = 'blank line' |
- |
- |
-class Token(object): |
- """Token class for intelligent text splitting. |
- |
- The token class represents a string of characters and an identifying type. |
- |
- Attributes: |
- type: The type of token. |
- string: The characters the token comprises. |
- length: The length of the token. |
- line: The text of the line the token is found in. |
- line_number: The number of the line the token is found in. |
- values: Dictionary of values returned from the tokens regex match. |
- previous: The token before this one. |
- next: The token after this one. |
- start_index: The character index in the line where this token starts. |
- attached_object: Object containing more information about this token. |
- metadata: Object containing metadata about this token. Must be added by |
- a separate metadata pass. |
- """ |
- |
- def __init__(self, string, token_type, line, line_number, values=None, |
- orig_line_number=None): |
- """Creates a new Token object. |
- |
- Args: |
- string: The string of input the token contains. |
- token_type: The type of token. |
- line: The text of the line this token is in. |
- line_number: The line number of the token. |
- values: A dict of named values within the token. For instance, a |
- function declaration may have a value called 'name' which captures the |
- name of the function. |
- orig_line_number: The line number of the original file this token comes |
- from. This should be only set during the tokenization process. For newly |
- created error fix tokens after that, it should be None. |
- """ |
- self.type = token_type |
- self.string = string |
- self.length = len(string) |
- self.line = line |
- self.line_number = line_number |
- self.orig_line_number = orig_line_number |
- self.values = values |
- self.is_deleted = False |
- |
- # These parts can only be computed when the file is fully tokenized |
- self.previous = None |
- self.next = None |
- self.start_index = None |
- |
- # This part is set in statetracker.py |
- # TODO(robbyw): Wrap this in to metadata |
- self.attached_object = None |
- |
- # This part is set in *metadatapass.py |
- self.metadata = None |
- |
- def IsFirstInLine(self): |
- """Tests if this token is the first token in its line. |
- |
- Returns: |
- Whether the token is the first token in its line. |
- """ |
- return not self.previous or self.previous.line_number != self.line_number |
- |
- def IsLastInLine(self): |
- """Tests if this token is the last token in its line. |
- |
- Returns: |
- Whether the token is the last token in its line. |
- """ |
- return not self.next or self.next.line_number != self.line_number |
- |
- def IsType(self, token_type): |
- """Tests if this token is of the given type. |
- |
- Args: |
- token_type: The type to test for. |
- |
- Returns: |
- True if the type of this token matches the type passed in. |
- """ |
- return self.type == token_type |
- |
- def IsAnyType(self, *token_types): |
- """Tests if this token is any of the given types. |
- |
- Args: |
- token_types: The types to check. Also accepts a single array. |
- |
- Returns: |
- True if the type of this token is any of the types passed in. |
- """ |
- if not isinstance(token_types[0], basestring): |
- return self.type in token_types[0] |
- else: |
- return self.type in token_types |
- |
- def __repr__(self): |
- return '<Token: %s, "%s", %r, %d, %r>' % (self.type, self.string, |
- self.values, self.line_number, |
- self.metadata) |
- |
- def __iter__(self): |
- """Returns a token iterator.""" |
- node = self |
- while node: |
- yield node |
- node = node.next |
- |
- def __reversed__(self): |
- """Returns a reverse-direction token iterator.""" |
- node = self |
- while node: |
- yield node |
- node = node.previous |