| Index: third_party/closure_linter/closure_linter/common/tokens.py
|
| diff --git a/third_party/closure_linter/closure_linter/common/tokens.py b/third_party/closure_linter/closure_linter/common/tokens.py
|
| deleted file mode 100755
|
| index 4703998752b03670ba5f3718483fb97fbaf77203..0000000000000000000000000000000000000000
|
| --- a/third_party/closure_linter/closure_linter/common/tokens.py
|
| +++ /dev/null
|
| @@ -1,145 +0,0 @@
|
| -#!/usr/bin/env python
|
| -#
|
| -# Copyright 2008 The Closure Linter Authors. All Rights Reserved.
|
| -#
|
| -# Licensed under the Apache License, Version 2.0 (the "License");
|
| -# you may not use this file except in compliance with the License.
|
| -# You may obtain a copy of the License at
|
| -#
|
| -# http://www.apache.org/licenses/LICENSE-2.0
|
| -#
|
| -# Unless required by applicable law or agreed to in writing, software
|
| -# distributed under the License is distributed on an "AS-IS" BASIS,
|
| -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| -# See the License for the specific language governing permissions and
|
| -# limitations under the License.
|
| -
|
| -"""Classes to represent tokens and positions within them."""
|
| -
|
| -__author__ = ('robbyw@google.com (Robert Walker)',
|
| - 'ajp@google.com (Andy Perelson)')
|
| -
|
| -
|
| -class TokenType(object):
|
| - """Token types common to all languages."""
|
| - NORMAL = 'normal'
|
| - WHITESPACE = 'whitespace'
|
| - BLANK_LINE = 'blank line'
|
| -
|
| -
|
| -class Token(object):
|
| - """Token class for intelligent text splitting.
|
| -
|
| - The token class represents a string of characters and an identifying type.
|
| -
|
| - Attributes:
|
| - type: The type of token.
|
| - string: The characters the token comprises.
|
| - length: The length of the token.
|
| - line: The text of the line the token is found in.
|
| - line_number: The number of the line the token is found in.
|
| - values: Dictionary of values returned from the tokens regex match.
|
| - previous: The token before this one.
|
| - next: The token after this one.
|
| - start_index: The character index in the line where this token starts.
|
| - attached_object: Object containing more information about this token.
|
| - metadata: Object containing metadata about this token. Must be added by
|
| - a separate metadata pass.
|
| - """
|
| -
|
| - def __init__(self, string, token_type, line, line_number, values=None,
|
| - orig_line_number=None):
|
| - """Creates a new Token object.
|
| -
|
| - Args:
|
| - string: The string of input the token contains.
|
| - token_type: The type of token.
|
| - line: The text of the line this token is in.
|
| - line_number: The line number of the token.
|
| - values: A dict of named values within the token. For instance, a
|
| - function declaration may have a value called 'name' which captures the
|
| - name of the function.
|
| - orig_line_number: The line number of the original file this token comes
|
| - from. This should be only set during the tokenization process. For newly
|
| - created error fix tokens after that, it should be None.
|
| - """
|
| - self.type = token_type
|
| - self.string = string
|
| - self.length = len(string)
|
| - self.line = line
|
| - self.line_number = line_number
|
| - self.orig_line_number = orig_line_number
|
| - self.values = values
|
| - self.is_deleted = False
|
| -
|
| - # These parts can only be computed when the file is fully tokenized
|
| - self.previous = None
|
| - self.next = None
|
| - self.start_index = None
|
| -
|
| - # This part is set in statetracker.py
|
| - # TODO(robbyw): Wrap this in to metadata
|
| - self.attached_object = None
|
| -
|
| - # This part is set in *metadatapass.py
|
| - self.metadata = None
|
| -
|
| - def IsFirstInLine(self):
|
| - """Tests if this token is the first token in its line.
|
| -
|
| - Returns:
|
| - Whether the token is the first token in its line.
|
| - """
|
| - return not self.previous or self.previous.line_number != self.line_number
|
| -
|
| - def IsLastInLine(self):
|
| - """Tests if this token is the last token in its line.
|
| -
|
| - Returns:
|
| - Whether the token is the last token in its line.
|
| - """
|
| - return not self.next or self.next.line_number != self.line_number
|
| -
|
| - def IsType(self, token_type):
|
| - """Tests if this token is of the given type.
|
| -
|
| - Args:
|
| - token_type: The type to test for.
|
| -
|
| - Returns:
|
| - True if the type of this token matches the type passed in.
|
| - """
|
| - return self.type == token_type
|
| -
|
| - def IsAnyType(self, *token_types):
|
| - """Tests if this token is any of the given types.
|
| -
|
| - Args:
|
| - token_types: The types to check. Also accepts a single array.
|
| -
|
| - Returns:
|
| - True if the type of this token is any of the types passed in.
|
| - """
|
| - if not isinstance(token_types[0], basestring):
|
| - return self.type in token_types[0]
|
| - else:
|
| - return self.type in token_types
|
| -
|
| - def __repr__(self):
|
| - return '<Token: %s, "%s", %r, %d, %r>' % (self.type, self.string,
|
| - self.values, self.line_number,
|
| - self.metadata)
|
| -
|
| - def __iter__(self):
|
| - """Returns a token iterator."""
|
| - node = self
|
| - while node:
|
| - yield node
|
| - node = node.next
|
| -
|
| - def __reversed__(self):
|
| - """Returns a reverse-direction token iterator."""
|
| - node = self
|
| - while node:
|
| - yield node
|
| - node = node.previous
|
|
|