closure_linter: 2.3.4 => 2.3.14

third_party/closure_linter/closure_linter/tokenutil.py

Index: third_party/closure_linter/closure_linter/tokenutil.py
diff --git a/third_party/closure_linter/closure_linter/tokenutil.py b/third_party/closure_linter/closure_linter/tokenutil.py
index 92ff16b317ae4187a596ddf9dfec9e30e713396c..8b5dbe1a9a3202b0730715d751f7084b92ec72fc 100755
--- a/third_party/closure_linter/closure_linter/tokenutil.py
+++ b/third_party/closure_linter/closure_linter/tokenutil.py
@@ -20,12 +20,13 @@ __author__ = ('robbyw@google.com (Robert Walker)',
               'ajp@google.com (Andy Perelson)')
 
 import copy
+import StringIO
 
-from closure_linter import javascripttokens
 from closure_linter.common import tokens
+from closure_linter.javascripttokens import JavaScriptToken
+from closure_linter.javascripttokens import JavaScriptTokenType
 
 # Shorthand
-JavaScriptToken = javascripttokens.JavaScriptToken
 Type = tokens.TokenType
 
 
@@ -214,6 +215,13 @@ def DeleteToken(token):
   Args:
     token: The token to delete
   """
+  # When deleting a token, we do not update the deleted token itself to make
+  # sure the previous and next pointers are still pointing to tokens which are
+  # not deleted.  Also it is very hard to keep track of all previously deleted
+  # tokens to update them when their pointers become invalid.  So we add this
+  # flag that any token linked list iteration logic can skip deleted node safely
+  # when its current token is deleted.
+  token.is_deleted = True
   if token.previous:
     token.previous.next = token.next
 
@@ -238,6 +246,47 @@ def DeleteTokens(token, token_count):
   DeleteToken(token)
 
 
+def InsertTokenBefore(new_token, token):
+  """Insert new_token before token.
+
+  Args:
+    new_token: A token to be added to the stream
+    token: A token already in the stream
+  """
+  new_token.next = token
+  new_token.previous = token.previous
+
+  new_token.metadata = copy.copy(token.metadata)
+
+  if new_token.IsCode():
+    old_last_code = token.metadata.last_code
+    following_token = token
+    while (following_token and
+           following_token.metadata.last_code == old_last_code):
+      following_token.metadata.last_code = new_token
+      following_token = following_token.next
+
+  token.previous = new_token
+  if new_token.previous:
+    new_token.previous.next = new_token
+
+  if new_token.start_index is None:
+    if new_token.line_number == token.line_number:
+      new_token.start_index = token.start_index
+    else:
+      previous_token = new_token.previous
+      if previous_token:
+        new_token.start_index = (previous_token.start_index +
+                                 len(previous_token.string))
+      else:
+        new_token.start_index = 0
+
+    iterator = new_token.next
+    while iterator and iterator.line_number == new_token.line_number:
+      iterator.start_index += len(new_token.string)
+      iterator = iterator.next
+
+
 def InsertTokenAfter(new_token, token):
   """Insert new_token after token.
 
@@ -372,3 +421,274 @@ def Compare(token1, token2):
     return token1.line_number - token2.line_number
   else:
     return token1.start_index - token2.start_index
+
+
+def GoogScopeOrNoneFromStartBlock(token):
+  """Determines if the given START_BLOCK is part of a goog.scope statement.
+
+  Args:
+    token: A token of type START_BLOCK.
+
+  Returns:
+    The goog.scope function call token, or None if such call doesn't exist.
+  """
+  if token.type != JavaScriptTokenType.START_BLOCK:
+    return None
+
+  # Search for a goog.scope statement, which will be 5 tokens before the
+  # block. Illustration of the tokens found prior to the start block:
+  # goog.scope(function() {
+  #      5    4    3   21 ^
+
+  maybe_goog_scope = token
+  for unused_i in xrange(5):
+    maybe_goog_scope = (maybe_goog_scope.previous if maybe_goog_scope and
+                        maybe_goog_scope.previous else None)
+  if maybe_goog_scope and maybe_goog_scope.string == 'goog.scope':
+    return maybe_goog_scope
+
+
+def GetTokenRange(start_token, end_token):
+  """Returns a list of tokens between the two given, inclusive.
+
+  Args:
+    start_token: Start token in the range.
+    end_token: End token in the range.
+
+  Returns:
+    A list of tokens, in order, from start_token to end_token (including start
+    and end).  Returns none if the tokens do not describe a valid range.
+  """
+
+  token_range = []
+  token = start_token
+
+  while token:
+    token_range.append(token)
+
+    if token == end_token:
+      return token_range
+
+    token = token.next
+
+
+def TokensToString(token_iterable):
+  """Convert a number of tokens into a string.
+
+  Newlines will be inserted whenever the line_number of two neighboring
+  strings differ.
+
+  Args:
+    token_iterable: The tokens to turn to a string.
+
+  Returns:
+    A string representation of the given tokens.
+  """
+
+  buf = StringIO.StringIO()
+  token_list = list(token_iterable)
+  if not token_list:
+    return ''
+
+  line_number = token_list[0].line_number
+
+  for token in token_list:
+
+    while line_number < token.line_number:
+      line_number += 1
+      buf.write('\n')
+
+    if line_number > token.line_number:
+      line_number = token.line_number
+      buf.write('\n')
+
+    buf.write(token.string)
+
+  return buf.getvalue()
+
+
+def GetPreviousCodeToken(token):
+  """Returns the code token before the specified token.
+
+  Args:
+    token: A token.
+
+  Returns:
+    The code token before the specified token or None if no such token
+    exists.
+  """
+
+  return CustomSearch(
+      token,
+      lambda t: t and t.type not in JavaScriptTokenType.NON_CODE_TYPES,
+      reverse=True)
+
+
+def GetNextCodeToken(token):
+  """Returns the next code token after the specified token.
+
+  Args:
+    token: A token.
+
+  Returns:
+    The next code token after the specified token or None if no such token
+    exists.
+  """
+
+  return CustomSearch(
+      token,
+      lambda t: t and t.type not in JavaScriptTokenType.NON_CODE_TYPES,
+      reverse=False)
+
+
+def GetIdentifierStart(token):
+  """Returns the first token in an identifier.
+
+  Given a token which is part of an identifier, returns the token at the start
+  of the identifier.
+
+  Args:
+    token: A token which is part of an identifier.
+
+  Returns:
+    The token at the start of the identifier or None if the identifier was not
+    of the form 'a.b.c' (e.g. "['a']['b'].c").
+  """
+
+  start_token = token
+  previous_code_token = GetPreviousCodeToken(token)
+
+  while (previous_code_token and (
+      previous_code_token.IsType(JavaScriptTokenType.IDENTIFIER) or
+      _IsDot(previous_code_token))):
+    start_token = previous_code_token
+    previous_code_token = GetPreviousCodeToken(previous_code_token)
+
+  if _IsDot(start_token):
+    return None
+
+  return start_token
+
+
+def GetIdentifierForToken(token):
+  """Get the symbol specified by a token.
+
+  Given a token, this function additionally concatenates any parts of an
+  identifying symbol being identified that are split by whitespace or a
+  newline.
+
+  The function will return None if the token is not the first token of an
+  identifier.
+
+  Args:
+    token: The first token of a symbol.
+
+  Returns:
+    The whole symbol, as a string.
+  """
+
+  # Search backward to determine if this token is the first token of the
+  # identifier. If it is not the first token, return None to signal that this
+  # token should be ignored.
+  prev_token = token.previous
+  while prev_token:
+    if (prev_token.IsType(JavaScriptTokenType.IDENTIFIER) or
+        _IsDot(prev_token)):
+      return None
+
+    if (prev_token.IsType(tokens.TokenType.WHITESPACE) or
+        prev_token.IsAnyType(JavaScriptTokenType.COMMENT_TYPES)):
+      prev_token = prev_token.previous
+    else:
+      break
+
+  # A "function foo()" declaration.
+  if token.type is JavaScriptTokenType.FUNCTION_NAME:
+    return token.string
+
+  # A "var foo" declaration (if the previous token is 'var')
+  previous_code_token = GetPreviousCodeToken(token)
+
+  if previous_code_token and previous_code_token.IsKeyword('var'):
+    return token.string
+
+  # Otherwise, this is potentially a namespaced (goog.foo.bar) identifier that
+  # could span multiple lines or be broken up by whitespace.  We need
+  # to concatenate.
+  identifier_types = set([
+      JavaScriptTokenType.IDENTIFIER,
+      JavaScriptTokenType.SIMPLE_LVALUE
+      ])
+
+  assert token.type in identifier_types
+
+  # Start with the first token
+  symbol_tokens = [token]
+
+  if token.next:
+    for t in token.next:
+      last_symbol_token = symbol_tokens[-1]
+
+      # An identifier is part of the previous symbol if it has a trailing
+      # dot.
+      if t.type in identifier_types:
+        if last_symbol_token.string.endswith('.'):
+          symbol_tokens.append(t)
+          continue
+        else:
+          break
+
+      # A dot is part of the previous symbol if it does not have a trailing
+      # dot.
+      if _IsDot(t):
+        if not last_symbol_token.string.endswith('.'):
+          symbol_tokens.append(t)
+          continue
+        else:
+          break
+
+      # Skip any whitespace
+      if t.type in JavaScriptTokenType.NON_CODE_TYPES:
+        continue
+
+      # This is the end of the identifier. Stop iterating.
+      break
+
+  if symbol_tokens:
+    return ''.join([t.string for t in symbol_tokens])
+
+
+def GetStringAfterToken(token):
+  """Get string after token.
+
+  Args:
+    token: Search will be done after this token.
+
+  Returns:
+    String if found after token else None (empty string will also
+    return None).
+
+  Search until end of string as in case of empty string Type.STRING_TEXT is not
+  present/found and don't want to return next string.
+  E.g.
+  a = '';
+  b = 'test';
+  When searching for string after 'a' if search is not limited by end of string
+  then it will return 'test' which is not desirable as there is a empty string
+  before that.
+
+  This will return None for cases where string is empty or no string found
+  as in both cases there is no Type.STRING_TEXT.
+  """
+  string_token = SearchUntil(token, JavaScriptTokenType.STRING_TEXT,
+                             [JavaScriptTokenType.SINGLE_QUOTE_STRING_END,
+                              JavaScriptTokenType.DOUBLE_QUOTE_STRING_END])
+  if string_token:
+    return string_token.string
+  else:
+    return None
+
+
+def _IsDot(token):
+  """Whether the token represents a "dot" operator (foo.bar)."""
+  return token.type is tokens.TokenType.NORMAL and token.string == '.'