Index: sky/tools/webkitpy/thirdparty/coverage/phystokens.py |
diff --git a/sky/tools/webkitpy/thirdparty/coverage/phystokens.py b/sky/tools/webkitpy/thirdparty/coverage/phystokens.py |
deleted file mode 100644 |
index fc4f2c9057b14e7c5ac79d4af2edfdb5df6139c9..0000000000000000000000000000000000000000 |
--- a/sky/tools/webkitpy/thirdparty/coverage/phystokens.py |
+++ /dev/null |
@@ -1,108 +0,0 @@ |
-"""Better tokenizing for coverage.py.""" |
- |
-import keyword, re, token, tokenize |
-from coverage.backward import StringIO # pylint: disable=W0622 |
- |
-def phys_tokens(toks): |
- """Return all physical tokens, even line continuations. |
- |
- tokenize.generate_tokens() doesn't return a token for the backslash that |
- continues lines. This wrapper provides those tokens so that we can |
- re-create a faithful representation of the original source. |
- |
- Returns the same values as generate_tokens() |
- |
- """ |
- last_line = None |
- last_lineno = -1 |
- last_ttype = None |
- for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks: |
- if last_lineno != elineno: |
- if last_line and last_line[-2:] == "\\\n": |
- # We are at the beginning of a new line, and the last line |
- # ended with a backslash. We probably have to inject a |
- # backslash token into the stream. Unfortunately, there's more |
- # to figure out. This code:: |
- # |
- # usage = """\ |
- # HEY THERE |
- # """ |
- # |
- # triggers this condition, but the token text is:: |
- # |
- # '"""\\\nHEY THERE\n"""' |
- # |
- # so we need to figure out if the backslash is already in the |
- # string token or not. |
- inject_backslash = True |
- if last_ttype == tokenize.COMMENT: |
- # Comments like this \ |
- # should never result in a new token. |
- inject_backslash = False |
- elif ttype == token.STRING: |
- if "\n" in ttext and ttext.split('\n', 1)[0][-1] == '\\': |
- # It's a multiline string and the first line ends with |
- # a backslash, so we don't need to inject another. |
- inject_backslash = False |
- if inject_backslash: |
- # Figure out what column the backslash is in. |
- ccol = len(last_line.split("\n")[-2]) - 1 |
- # Yield the token, with a fake token type. |
- yield ( |
- 99999, "\\\n", |
- (slineno, ccol), (slineno, ccol+2), |
- last_line |
- ) |
- last_line = ltext |
- last_ttype = ttype |
- yield ttype, ttext, (slineno, scol), (elineno, ecol), ltext |
- last_lineno = elineno |
- |
- |
-def source_token_lines(source): |
- """Generate a series of lines, one for each line in `source`. |
- |
- Each line is a list of pairs, each pair is a token:: |
- |
- [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ] |
- |
- Each pair has a token class, and the token text. |
- |
- If you concatenate all the token texts, and then join them with newlines, |
- you should have your original `source` back, with two differences: |
- trailing whitespace is not preserved, and a final line with no newline |
- is indistinguishable from a final line with a newline. |
- |
- """ |
- ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL] |
- line = [] |
- col = 0 |
- source = source.expandtabs(8).replace('\r\n', '\n') |
- tokgen = tokenize.generate_tokens(StringIO(source).readline) |
- for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen): |
- mark_start = True |
- for part in re.split('(\n)', ttext): |
- if part == '\n': |
- yield line |
- line = [] |
- col = 0 |
- mark_end = False |
- elif part == '': |
- mark_end = False |
- elif ttype in ws_tokens: |
- mark_end = False |
- else: |
- if mark_start and scol > col: |
- line.append(("ws", " " * (scol - col))) |
- mark_start = False |
- tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3] |
- if ttype == token.NAME and keyword.iskeyword(ttext): |
- tok_class = "key" |
- line.append((tok_class, part)) |
- mark_end = True |
- scol = 0 |
- if mark_end: |
- col = ecol |
- |
- if line: |
- yield line |