Index: tools/telemetry/catapult_base/refactor/snippet.py |
diff --git a/tools/telemetry/catapult_base/refactor/snippet.py b/tools/telemetry/catapult_base/refactor/snippet.py |
deleted file mode 100644 |
index ecb688af9004038866ca44a1335dd003c02356ab..0000000000000000000000000000000000000000 |
--- a/tools/telemetry/catapult_base/refactor/snippet.py |
+++ /dev/null |
@@ -1,242 +0,0 @@ |
-# Copyright 2015 The Chromium Authors. All rights reserved. |
-# Use of this source code is governed by a BSD-style license that can be |
-# found in the LICENSE file. |
- |
-import parser |
-import symbol |
-import sys |
-import token |
-import tokenize |
- |
-from catapult_base.refactor import offset_token |
- |
- |
-class Snippet(object): |
- """A node in the Python parse tree. |
- |
- The Python grammar is defined at: |
- https://docs.python.org/2/reference/grammar.html |
- |
- There are two types of Snippets: |
- TokenSnippets are leaf nodes containing actual text. |
- Symbols are internal nodes representing higher-level groupings, and are |
- defined by the left-hand sides of the BNFs in the above link. |
- """ |
- @property |
- def type(self): |
- raise NotImplementedError() |
- |
- @property |
- def type_name(self): |
- raise NotImplementedError() |
- |
- @property |
- def children(self): |
- """Return a list of this node's children.""" |
- raise NotImplementedError() |
- |
- @property |
- def tokens(self): |
- """Return a tuple of the tokens this Snippet contains.""" |
- raise NotImplementedError() |
- |
- def PrintTree(self, indent=0, stream=sys.stdout): |
- """Spew a pretty-printed parse tree. Mostly useful for debugging.""" |
- raise NotImplementedError() |
- |
- def __str__(self): |
- return offset_token.Untokenize(self.tokens) |
- |
- def FindAll(self, snippet_type): |
- if isinstance(snippet_type, int): |
- if self.type == snippet_type: |
- yield self |
- else: |
- if isinstance(self, snippet_type): |
- yield self |
- |
- for child in self.children: |
- for snippet in child.FindAll(snippet_type): |
- yield snippet |
- |
- def FindChild(self, snippet_type, **kwargs): |
- for child in self.children: |
- if isinstance(snippet_type, int): |
- if child.type != snippet_type: |
- continue |
- else: |
- if not isinstance(child, snippet_type): |
- continue |
- |
- for attribute, value in kwargs: |
- if getattr(child, attribute) != value: |
- break |
- else: |
- return child |
- raise ValueError('%s is not in %s. Children are: %s' % |
- (snippet_type, self, self.children)) |
- |
- def FindChildren(self, snippet_type): |
- if isinstance(snippet_type, int): |
- for child in self.children: |
- if child.type == snippet_type: |
- yield child |
- else: |
- for child in self.children: |
- if isinstance(child, snippet_type): |
- yield child |
- |
- |
-class TokenSnippet(Snippet): |
- """A Snippet containing a list of tokens. |
- |
- A list of tokens may start with any number of comments and non-terminating |
- newlines, but must end with a syntactically meaningful token. |
- """ |
- def __init__(self, token_type, tokens): |
- # For operators and delimiters, the TokenSnippet's type may be more specific |
- # than the type of the constituent token. E.g. the TokenSnippet type is |
- # token.DOT, but the token type is token.OP. This is because the parser |
- # has more context than the tokenizer. |
- self._type = token_type |
- self._tokens = tokens |
- self._modified = False |
- |
- @classmethod |
- def Create(cls, token_type, string, offset=(0, 0)): |
- return cls(token_type, |
- [offset_token.OffsetToken(token_type, string, offset)]) |
- |
- @property |
- def type(self): |
- return self._type |
- |
- @property |
- def type_name(self): |
- return token.tok_name[self.type] |
- |
- @property |
- def value(self): |
- return self._tokens[-1].string |
- |
- @value.setter |
- def value(self, value): |
- self._tokens[-1].string = value |
- self._modified = True |
- |
- @property |
- def children(self): |
- return [] |
- |
- @property |
- def tokens(self): |
- return tuple(self._tokens) |
- |
- @property |
- def modified(self): |
- return self._modified |
- |
- def PrintTree(self, indent=0, stream=sys.stdout): |
- stream.write(' ' * indent) |
- if not self.tokens: |
- print >> stream, self.type_name |
- return |
- |
- print >> stream, '%-4s' % self.type_name, repr(self.tokens[0].string) |
- for tok in self.tokens[1:]: |
- stream.write(' ' * indent) |
- print >> stream, ' ' * max(len(self.type_name), 4), repr(tok.string) |
- |
- |
-class Symbol(Snippet): |
- """A Snippet containing sub-Snippets. |
- |
- The possible types and type_names are defined in Python's symbol module.""" |
- def __init__(self, symbol_type, children): |
- self._type = symbol_type |
- self._children = children |
- |
- @property |
- def type(self): |
- return self._type |
- |
- @property |
- def type_name(self): |
- return symbol.sym_name[self.type] |
- |
- @property |
- def children(self): |
- return self._children |
- |
- @children.setter |
- def children(self, value): # pylint: disable=arguments-differ |
- self._children = value |
- |
- @property |
- def tokens(self): |
- tokens = [] |
- for child in self.children: |
- tokens += child.tokens |
- return tuple(tokens) |
- |
- @property |
- def modified(self): |
- return any(child.modified for child in self.children) |
- |
- def PrintTree(self, indent=0, stream=sys.stdout): |
- stream.write(' ' * indent) |
- |
- # If there's only one child, collapse it onto the same line. |
- node = self |
- while len(node.children) == 1 and len(node.children[0].children) == 1: |
- print >> stream, node.type_name, |
- node = node.children[0] |
- |
- print >> stream, node.type_name |
- for child in node.children: |
- child.PrintTree(indent+2, stream) |
- |
- |
-def Snippetize(f): |
- """Return the syntax tree of the given file.""" |
- f.seek(0) |
- syntax_tree = parser.st2list(parser.suite(f.read())) |
- tokens = offset_token.Tokenize(f) |
- |
- snippet = _SnippetizeNode(syntax_tree, tokens) |
- assert not tokens |
- return snippet |
- |
- |
-def _SnippetizeNode(node, tokens): |
- # The parser module gives a syntax tree that discards comments, |
- # non-terminating newlines, and whitespace information. Use the tokens given |
- # by the tokenize module to annotate the syntax tree with the information |
- # needed to exactly reproduce the original source code. |
- node_type = node[0] |
- |
- if node_type >= token.NT_OFFSET: |
- # Symbol. |
- children = tuple(_SnippetizeNode(child, tokens) for child in node[1:]) |
- return Symbol(node_type, children) |
- else: |
- # Token. |
- grabbed_tokens = [] |
- while tokens and ( |
- tokens[0].type == tokenize.COMMENT or tokens[0].type == tokenize.NL): |
- grabbed_tokens.append(tokens.popleft()) |
- |
- # parser has 2 NEWLINEs right before the end. |
- # tokenize has 0 or 1 depending on if the file has one. |
- # Create extra nodes without consuming tokens to account for this. |
- if node_type == token.NEWLINE: |
- for tok in tokens: |
- if tok.type == token.ENDMARKER: |
- return TokenSnippet(node_type, grabbed_tokens) |
- if tok.type != token.DEDENT: |
- break |
- |
- assert tokens[0].type == token.OP or node_type == tokens[0].type |
- |
- grabbed_tokens.append(tokens.popleft()) |
- return TokenSnippet(node_type, grabbed_tokens) |