Remove closure_linter from Chrome

third_party/closure_linter/closure_linter/common/tokenizer.py

-#!/usr/bin/env python
-#
-# Copyright 2007 The Closure Linter Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS-IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Regular expression based lexer."""
-
-__author__ = ('robbyw@google.com (Robert Walker)',
-              'ajp@google.com (Andy Perelson)')
-
-from closure_linter.common import tokens
-
-# Shorthand
-Type = tokens.TokenType
-
-
-class Tokenizer(object):
-  """General purpose tokenizer.
-
-  Attributes:
-    mode: The latest mode of the tokenizer.  This allows patterns to distinguish
-        if they are mid-comment, mid-parameter list, etc.
-    matchers: Dictionary of modes to sequences of matchers that define the
-        patterns to check at any given time.
-    default_types: Dictionary of modes to types, defining what type to give
-        non-matched text when in the given mode.  Defaults to Type.NORMAL.
-  """
-
-  def __init__(self, starting_mode, matchers, default_types):
-    """Initialize the tokenizer.
-
-    Args:
-      starting_mode: Mode to start in.
-      matchers: Dictionary of modes to sequences of matchers that defines the
-          patterns to check at any given time.
-      default_types: Dictionary of modes to types, defining what type to give
-          non-matched text when in the given mode.  Defaults to Type.NORMAL.
-    """
-    self.__starting_mode = starting_mode
-    self.matchers = matchers
-    self.default_types = default_types
-
-  def TokenizeFile(self, file):
-    """Tokenizes the given file.
-
-    Args:
-      file: An iterable that yields one line of the file at a time.
-
-    Returns:
-      The first token in the file
-    """
-    # The current mode.
-    self.mode = self.__starting_mode
-    # The first token in the stream.
-    self.__first_token = None
-    # The last token added to the token stream.
-    self.__last_token = None
-    # The current line number.
-    self.__line_number = 0
-
-    for line in file:
-      self.__line_number += 1
-      self.__TokenizeLine(line)
-
-    return self.__first_token
-
-  def _CreateToken(self, string, token_type, line, line_number, values=None):
-    """Creates a new Token object (or subclass).
-
-    Args:
-      string: The string of input the token represents.
-      token_type: The type of token.
-      line: The text of the line this token is in.
-      line_number: The line number of the token.
-      values: A dict of named values within the token.  For instance, a
-        function declaration may have a value called 'name' which captures the
-        name of the function.
-
-    Returns:
-      The newly created Token object.
-    """
-    return tokens.Token(string, token_type, line, line_number, values,
-                        line_number)
-
-  def __TokenizeLine(self, line):
-    """Tokenizes the given line.
-
-    Args:
-      line: The contents of the line.
-    """
-    string = line.rstrip('\n\r\f')
-    line_number = self.__line_number
-    self.__start_index = 0
-
-    if not string:
-      self.__AddToken(self._CreateToken('', Type.BLANK_LINE, line, line_number))
-      return
-
-    normal_token = ''
-    index = 0
-    while index < len(string):
-      for matcher in self.matchers[self.mode]:
-        if matcher.line_start and index > 0:
-          continue
-
-        match = matcher.regex.match(string, index)
-
-        if match:
-          if normal_token:
-            self.__AddToken(
-                self.__CreateNormalToken(self.mode, normal_token, line,
-                                         line_number))
-            normal_token = ''
-
-          # Add the match.
-          self.__AddToken(self._CreateToken(match.group(), matcher.type, line,
-                                            line_number, match.groupdict()))
-
-          # Change the mode to the correct one for after this match.
-          self.mode = matcher.result_mode or self.mode
-
-          # Shorten the string to be matched.
-          index = match.end()
-
-          break
-
-      else:
-        # If the for loop finishes naturally (i.e. no matches) we just add the
-        # first character to the string of consecutive non match characters.
-        # These will constitute a NORMAL token.
-        if string:
-          normal_token += string[index:index + 1]
-          index += 1
-
-    if normal_token:
-      self.__AddToken(
-          self.__CreateNormalToken(self.mode, normal_token, line, line_number))
-
-  def __CreateNormalToken(self, mode, string, line, line_number):
-    """Creates a normal token.
-
-    Args:
-      mode: The current mode.
-      string: The string to tokenize.
-      line: The line of text.
-      line_number: The line number within the file.
-
-    Returns:
-      A Token object, of the default type for the current mode.
-    """
-    type = Type.NORMAL
-    if mode in self.default_types:
-      type = self.default_types[mode]
-    return self._CreateToken(string, type, line, line_number)
-
-  def __AddToken(self, token):
-    """Add the given token to the token stream.
-
-    Args:
-      token: The token to add.
-    """
-    # Store the first token, or point the previous token to this one.
-    if not self.__first_token:
-      self.__first_token = token
-    else:
-      self.__last_token.next = token
-
-    # Establish the doubly linked list
-    token.previous = self.__last_token
-    self.__last_token = token
-
-    # Compute the character indices
-    token.start_index = self.__start_index
-    self.__start_index += token.length