Revert of Remove catapult_base from telemetry.

tools/telemetry/catapult_base/refactor/offset_token.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import itertools
+import token
+import tokenize
+
+
+def _Pairwise(iterable):
+  """s -> (None, s0), (s0, s1), (s1, s2), (s2, s3), ..."""
+  a, b = itertools.tee(iterable)
+  a = itertools.chain((None,), a)
+  return itertools.izip(a, b)
+
+
+class OffsetToken(object):
+  """A Python token with a relative position.
+
+  A token is represented by a type defined in Python's token module, a string
+  representing the content, and an offset. Using relative positions makes it
+  easy to insert and remove tokens.
+  """
+  def __init__(self, token_type, string, offset):
+    self._type = token_type
+    self._string = string
+    self._offset = offset
+
+  @property
+  def type(self):
+    return self._type
+
+  @property
+  def type_name(self):
+    return token.tok_name[self._type]
+
+  @property
+  def string(self):
+    return self._string
+
+  @string.setter
+  def string(self, value):
+    self._string = value
+
+  @property
+  def offset(self):
+    return self._offset
+
+  def __str__(self):
+    return str((self.type_name, self.string, self.offset))
+
+
+def Tokenize(f):
+  """Read tokens from a file-like object.
+
+  Args:
+    f: Any object that has a readline method.
+
+  Returns:
+    A collections.deque containing OffsetTokens. Deques are cheaper and easier
+    to manipulate sequentially than lists.
+  """
+  f.seek(0)
+  tokenize_tokens = tokenize.generate_tokens(f.readline)
+
+  offset_tokens = collections.deque()
+  for prev_token, next_token in _Pairwise(tokenize_tokens):
+    token_type, string, (srow, scol), _, _ = next_token
+    if not prev_token:
+      offset_tokens.append(OffsetToken(token_type, string, (0, 0)))
+    else:
+      erow, ecol = prev_token[3]
+      if erow == srow:
+        offset_tokens.append(OffsetToken(token_type, string, (0, scol-ecol)))
+      else:
+        offset_tokens.append(OffsetToken(token_type, string, (srow-erow, scol)))
+
+  return offset_tokens
+
+
+def Untokenize(offset_tokens):
+  """Return the string representation of an iterable of OffsetTokens."""
+  # Make a copy. Don't modify the original.
+  offset_tokens = collections.deque(offset_tokens)
+
+  # Strip leading NL tokens.
+  while offset_tokens[0].type == tokenize.NL:
+    offset_tokens.popleft()
+
+  # Strip leading vertical whitespace.
+  first_token = offset_tokens.popleft()
+  # Take care not to modify the existing token. Create a new one in its place.
+  first_token = OffsetToken(first_token.type, first_token.string,
+                            (0, first_token.offset[1]))
+  offset_tokens.appendleft(first_token)
+
+  # Convert OffsetTokens to tokenize tokens.
+  tokenize_tokens = []
+  row = 1
+  col = 0
+  for t in offset_tokens:
+    offset_row, offset_col = t.offset
+    if offset_row == 0:
+      col += offset_col
+    else:
+      row += offset_row
+      col = offset_col
+    tokenize_tokens.append((t.type, t.string, (row, col), (row, col), None))
+
+  # tokenize can't handle whitespace before line continuations.
+  # So add a space.
+  return tokenize.untokenize(tokenize_tokens).replace('\\\n', ' \\\n')