Check in the thirdparty libs needed for webkitpy.

Tools/Scripts/webkitpy/thirdparty/pep8.py

Index: Tools/Scripts/webkitpy/thirdparty/pep8.py
diff --git a/Tools/Scripts/webkitpy/thirdparty/pep8.py b/Tools/Scripts/webkitpy/thirdparty/pep8.py
new file mode 100755
index 0000000000000000000000000000000000000000..c31937039ad137c6d2166cedacc016babc522541
--- /dev/null
+++ b/Tools/Scripts/webkitpy/thirdparty/pep8.py
@@ -0,0 +1,1254 @@
+#!/usr/bin/python
+# pep8.py - Check Python source code formatting, according to PEP 8
+# Copyright (C) 2006 Johann C. Rocholl <johann@rocholl.net>
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Check Python source code formatting, according to PEP 8:
+http://www.python.org/dev/peps/pep-0008/
+
+For usage and a list of options, try this:
+$ python pep8.py -h
+
+This program and its regression test suite live here:
+http://github.com/jcrocholl/pep8
+
+Groups of errors and warnings:
+E errors
+W warnings
+100 indentation
+200 whitespace
+300 blank lines
+400 imports
+500 line length
+600 deprecation
+700 statements
+
+You can add checks to this program by writing plugins. Each plugin is
+a simple function that is called for each line of source code, either
+physical or logical.
+
+Physical line:
+- Raw line of text from the input file.
+
+Logical line:
+- Multi-line statements converted to a single line.
+- Stripped left and right.
+- Contents of strings replaced with 'xxx' of same length.
+- Comments removed.
+
+The check function requests physical or logical lines by the name of
+the first argument:
+
+def maximum_line_length(physical_line)
+def extraneous_whitespace(logical_line)
+def blank_lines(logical_line, blank_lines, indent_level, line_number)
+
+The last example above demonstrates how check plugins can request
+additional information with extra arguments. All attributes of the
+Checker object are available. Some examples:
+
+lines: a list of the raw lines from the input file
+tokens: the tokens that contribute to this logical line
+line_number: line number in the input file
+blank_lines: blank lines before this one
+indent_char: first indentation character in this file (' ' or '\t')
+indent_level: indentation (with tabs expanded to multiples of 8)
+previous_indent_level: indentation on previous line
+previous_logical: previous logical line
+
+The docstring of each check function shall be the relevant part of
+text from PEP 8. It is printed if the user enables --show-pep8.
+Several docstrings contain examples directly from the PEP 8 document.
+
+Okay: spam(ham[1], {eggs: 2})
+E201: spam( ham[1], {eggs: 2})
+
+These examples are verified automatically when pep8.py is run with the
+--doctest option. You can add examples for your own check functions.
+The format is simple: "Okay" or error/warning code followed by colon
+and space, the rest of the line is example source code. If you put 'r'
+before the docstring, you can use \n for newline, \t for tab and \s
+for space.
+
+"""
+
+__version__ = '0.5.0'
+
+import os
+import sys
+import re
+import time
+import inspect
+import tokenize
+from optparse import OptionParser
+from keyword import iskeyword
+from fnmatch import fnmatch
+
+DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git'
+DEFAULT_IGNORE = ['E24']
+
+INDENT_REGEX = re.compile(r'([ \t]*)')
+RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*(,)')
+SELFTEST_REGEX = re.compile(r'(Okay|[EW]\d{3}):\s(.*)')
+ERRORCODE_REGEX = re.compile(r'[EW]\d{3}')
+E301NOT_REGEX = re.compile(r'class |def |u?r?["\']')
+
+WHITESPACE = ' \t'
+
+BINARY_OPERATORS = ['**=', '*=', '+=', '-=', '!=', '<>',
+    '%=', '^=', '&=', '|=', '==', '/=', '//=', '>=', '<=', '>>=', '<<=',
+    '%',  '^',  '&',  '|',  '=',  '/',  '//',  '>',  '<',  '>>',  '<<']
+UNARY_OPERATORS = ['**', '*', '+', '-']
+OPERATORS = BINARY_OPERATORS + UNARY_OPERATORS
+
+options = None
+args = None
+
+
+##############################################################################
+# Plugins (check functions) for physical lines
+##############################################################################
+
+
+def tabs_or_spaces(physical_line, indent_char):
+    r"""
+    Never mix tabs and spaces.
+
+    The most popular way of indenting Python is with spaces only.  The
+    second-most popular way is with tabs only.  Code indented with a mixture
+    of tabs and spaces should be converted to using spaces exclusively.  When
+    invoking the Python command line interpreter with the -t option, it issues
+    warnings about code that illegally mixes tabs and spaces.  When using -tt
+    these warnings become errors.  These options are highly recommended!
+
+    Okay: if a == 0:\n        a = 1\n        b = 1
+    E101: if a == 0:\n        a = 1\n\tb = 1
+    """
+    indent = INDENT_REGEX.match(physical_line).group(1)
+    for offset, char in enumerate(indent):
+        if char != indent_char:
+            return offset, "E101 indentation contains mixed spaces and tabs"
+
+
+def tabs_obsolete(physical_line):
+    r"""
+    For new projects, spaces-only are strongly recommended over tabs.  Most
+    editors have features that make this easy to do.
+
+    Okay: if True:\n    return
+    W191: if True:\n\treturn
+    """
+    indent = INDENT_REGEX.match(physical_line).group(1)
+    if indent.count('\t'):
+        return indent.index('\t'), "W191 indentation contains tabs"
+
+
+def trailing_whitespace(physical_line):
+    """
+    JCR: Trailing whitespace is superfluous.
+
+    Okay: spam(1)
+    W291: spam(1)\s
+    """
+    physical_line = physical_line.rstrip('\n')    # chr(10), newline
+    physical_line = physical_line.rstrip('\r')    # chr(13), carriage return
+    physical_line = physical_line.rstrip('\x0c')  # chr(12), form feed, ^L
+    stripped = physical_line.rstrip()
+    if physical_line != stripped:
+        return len(stripped), "W291 trailing whitespace"
+
+
+def trailing_blank_lines(physical_line, lines, line_number):
+    r"""
+    JCR: Trailing blank lines are superfluous.
+
+    Okay: spam(1)
+    W391: spam(1)\n
+    """
+    if physical_line.strip() == '' and line_number == len(lines):
+        return 0, "W391 blank line at end of file"
+
+
+def missing_newline(physical_line):
+    """
+    JCR: The last line should have a newline.
+    """
+    if physical_line.rstrip() == physical_line:
+        return len(physical_line), "W292 no newline at end of file"
+
+
+def maximum_line_length(physical_line):
+    """
+    Limit all lines to a maximum of 79 characters.
+
+    There are still many devices around that are limited to 80 character
+    lines; plus, limiting windows to 80 characters makes it possible to have
+    several windows side-by-side.  The default wrapping on such devices looks
+    ugly.  Therefore, please limit all lines to a maximum of 79 characters.
+    For flowing long blocks of text (docstrings or comments), limiting the
+    length to 72 characters is recommended.
+    """
+    length = len(physical_line.rstrip())
+    if length > 79:
+        return 79, "E501 line too long (%d characters)" % length
+
+
+##############################################################################
+# Plugins (check functions) for logical lines
+##############################################################################
+
+
+def blank_lines(logical_line, blank_lines, indent_level, line_number,
+                previous_logical, blank_lines_before_comment):
+    r"""
+    Separate top-level function and class definitions with two blank lines.
+
+    Method definitions inside a class are separated by a single blank line.
+
+    Extra blank lines may be used (sparingly) to separate groups of related
+    functions.  Blank lines may be omitted between a bunch of related
+    one-liners (e.g. a set of dummy implementations).
+
+    Use blank lines in functions, sparingly, to indicate logical sections.
+
+    Okay: def a():\n    pass\n\n\ndef b():\n    pass
+    Okay: def a():\n    pass\n\n\n# Foo\n# Bar\n\ndef b():\n    pass
+
+    E301: class Foo:\n    b = 0\n    def bar():\n        pass
+    E302: def a():\n    pass\n\ndef b(n):\n    pass
+    E303: def a():\n    pass\n\n\n\ndef b(n):\n    pass
+    E303: def a():\n\n\n\n    pass
+    E304: @decorator\n\ndef a():\n    pass
+    """
+    if line_number == 1:
+        return  # Don't expect blank lines before the first line
+    max_blank_lines = max(blank_lines, blank_lines_before_comment)
+    if previous_logical.startswith('@'):
+        if max_blank_lines:
+            return 0, "E304 blank lines found after function decorator"
+    elif max_blank_lines > 2 or (indent_level and max_blank_lines == 2):
+        return 0, "E303 too many blank lines (%d)" % max_blank_lines
+    elif (logical_line.startswith('def ') or
+          logical_line.startswith('class ') or
+          logical_line.startswith('@')):
+        if indent_level:
+            if not (max_blank_lines or E301NOT_REGEX.match(previous_logical)):
+                return 0, "E301 expected 1 blank line, found 0"
+        elif max_blank_lines != 2:
+            return 0, "E302 expected 2 blank lines, found %d" % max_blank_lines
+
+
+def extraneous_whitespace(logical_line):
+    """
+    Avoid extraneous whitespace in the following situations:
+
+    - Immediately inside parentheses, brackets or braces.
+
+    - Immediately before a comma, semicolon, or colon.
+
+    Okay: spam(ham[1], {eggs: 2})
+    E201: spam( ham[1], {eggs: 2})
+    E201: spam(ham[ 1], {eggs: 2})
+    E201: spam(ham[1], { eggs: 2})
+    E202: spam(ham[1], {eggs: 2} )
+    E202: spam(ham[1 ], {eggs: 2})
+    E202: spam(ham[1], {eggs: 2 })
+
+    E203: if x == 4: print x, y; x, y = y , x
+    E203: if x == 4: print x, y ; x, y = y, x
+    E203: if x == 4 : print x, y; x, y = y, x
+    """
+    line = logical_line
+    for char in '([{':
+        found = line.find(char + ' ')
+        if found > -1:
+            return found + 1, "E201 whitespace after '%s'" % char
+    for char in '}])':
+        found = line.find(' ' + char)
+        if found > -1 and line[found - 1] != ',':
+            return found, "E202 whitespace before '%s'" % char
+    for char in ',;:':
+        found = line.find(' ' + char)
+        if found > -1:
+            return found, "E203 whitespace before '%s'" % char
+
+
+def missing_whitespace(logical_line):
+    """
+    JCR: Each comma, semicolon or colon should be followed by whitespace.
+
+    Okay: [a, b]
+    Okay: (3,)
+    Okay: a[1:4]
+    Okay: a[:4]
+    Okay: a[1:]
+    Okay: a[1:4:2]
+    E231: ['a','b']
+    E231: foo(bar,baz)
+    """
+    line = logical_line
+    for index in range(len(line) - 1):
+        char = line[index]
+        if char in ',;:' and line[index + 1] not in WHITESPACE:
+            before = line[:index]
+            if char == ':' and before.count('[') > before.count(']'):
+                continue  # Slice syntax, no space required
+            if char == ',' and line[index + 1] == ')':
+                continue  # Allow tuple with only one element: (3,)
+            return index, "E231 missing whitespace after '%s'" % char
+
+
+def indentation(logical_line, previous_logical, indent_char,
+                indent_level, previous_indent_level):
+    r"""
+    Use 4 spaces per indentation level.
+
+    For really old code that you don't want to mess up, you can continue to
+    use 8-space tabs.
+
+    Okay: a = 1
+    Okay: if a == 0:\n    a = 1
+    E111:   a = 1
+
+    Okay: for item in items:\n    pass
+    E112: for item in items:\npass
+
+    Okay: a = 1\nb = 2
+    E113: a = 1\n    b = 2
+    """
+    if indent_char == ' ' and indent_level % 4:
+        return 0, "E111 indentation is not a multiple of four"
+    indent_expect = previous_logical.endswith(':')
+    if indent_expect and indent_level <= previous_indent_level:
+        return 0, "E112 expected an indented block"
+    if indent_level > previous_indent_level and not indent_expect:
+        return 0, "E113 unexpected indentation"
+
+
+def whitespace_before_parameters(logical_line, tokens):
+    """
+    Avoid extraneous whitespace in the following situations:
+
+    - Immediately before the open parenthesis that starts the argument
+      list of a function call.
+
+    - Immediately before the open parenthesis that starts an indexing or
+      slicing.
+
+    Okay: spam(1)
+    E211: spam (1)
+
+    Okay: dict['key'] = list[index]
+    E211: dict ['key'] = list[index]
+    E211: dict['key'] = list [index]
+    """
+    prev_type = tokens[0][0]
+    prev_text = tokens[0][1]
+    prev_end = tokens[0][3]
+    for index in range(1, len(tokens)):
+        token_type, text, start, end, line = tokens[index]
+        if (token_type == tokenize.OP and
+            text in '([' and
+            start != prev_end and
+            prev_type == tokenize.NAME and
+            (index < 2 or tokens[index - 2][1] != 'class') and
+            (not iskeyword(prev_text))):
+            return prev_end, "E211 whitespace before '%s'" % text
+        prev_type = token_type
+        prev_text = text
+        prev_end = end
+
+
+def whitespace_around_operator(logical_line):
+    """
+    Avoid extraneous whitespace in the following situations:
+
+    - More than one space around an assignment (or other) operator to
+      align it with another.
+
+    Okay: a = 12 + 3
+    E221: a = 4  + 5
+    E222: a = 4 +  5
+    E223: a = 4\t+ 5
+    E224: a = 4 +\t5
+    """
+    line = logical_line
+    for operator in OPERATORS:
+        found = line.find('  ' + operator)
+        if found > -1:
+            return found, "E221 multiple spaces before operator"
+        found = line.find(operator + '  ')
+        if found > -1:
+            return found, "E222 multiple spaces after operator"
+        found = line.find('\t' + operator)
+        if found > -1:
+            return found, "E223 tab before operator"
+        found = line.find(operator + '\t')
+        if found > -1:
+            return found, "E224 tab after operator"
+
+
+def missing_whitespace_around_operator(logical_line, tokens):
+    r"""
+    - Always surround these binary operators with a single space on
+      either side: assignment (=), augmented assignment (+=, -= etc.),
+      comparisons (==, <, >, !=, <>, <=, >=, in, not in, is, is not),
+      Booleans (and, or, not).
+
+    - Use spaces around arithmetic operators.
+
+    Okay: i = i + 1
+    Okay: submitted += 1
+    Okay: x = x * 2 - 1
+    Okay: hypot2 = x * x + y * y
+    Okay: c = (a + b) * (a - b)
+    Okay: foo(bar, key='word', *args, **kwargs)
+    Okay: baz(**kwargs)
+    Okay: negative = -1
+    Okay: spam(-1)
+    Okay: alpha[:-i]
+    Okay: if not -5 < x < +5:\n    pass
+    Okay: lambda *args, **kw: (args, kw)
+
+    E225: i=i+1
+    E225: submitted +=1
+    E225: x = x*2 - 1
+    E225: hypot2 = x*x + y*y
+    E225: c = (a+b) * (a-b)
+    E225: c = alpha -4
+    E225: z = x **y
+    """
+    parens = 0
+    need_space = False
+    prev_type = tokenize.OP
+    prev_text = prev_end = None
+    for token_type, text, start, end, line in tokens:
+        if token_type in (tokenize.NL, tokenize.NEWLINE, tokenize.ERRORTOKEN):
+            # ERRORTOKEN is triggered by backticks in Python 3000
+            continue
+        if text in ('(', 'lambda'):
+            parens += 1
+        elif text == ')':
+            parens -= 1
+        if need_space:
+            if start == prev_end:
+                return prev_end, "E225 missing whitespace around operator"
+            need_space = False
+        elif token_type == tokenize.OP:
+            if text == '=' and parens:
+                # Allow keyword args or defaults: foo(bar=None).
+                pass
+            elif text in BINARY_OPERATORS:
+                need_space = True
+            elif text in UNARY_OPERATORS:
+                if ((prev_type != tokenize.OP or prev_text in '}])') and not
+                    (prev_type == tokenize.NAME and iskeyword(prev_text))):
+                    # Allow unary operators: -123, -x, +1.
+                    # Allow argument unpacking: foo(*args, **kwargs).
+                    need_space = True
+            if need_space and start == prev_end:
+                return prev_end, "E225 missing whitespace around operator"
+        prev_type = token_type
+        prev_text = text
+        prev_end = end
+
+
+def whitespace_around_comma(logical_line):
+    """
+    Avoid extraneous whitespace in the following situations:
+
+    - More than one space around an assignment (or other) operator to
+      align it with another.
+
+    JCR: This should also be applied around comma etc.
+    Note: these checks are disabled by default
+
+    Okay: a = (1, 2)
+    E241: a = (1,  2)
+    E242: a = (1,\t2)
+    """
+    line = logical_line
+    for separator in ',;:':
+        found = line.find(separator + '  ')
+        if found > -1:
+            return found + 1, "E241 multiple spaces after '%s'" % separator
+        found = line.find(separator + '\t')
+        if found > -1:
+            return found + 1, "E242 tab after '%s'" % separator
+
+
+def whitespace_around_named_parameter_equals(logical_line):
+    """
+    Don't use spaces around the '=' sign when used to indicate a
+    keyword argument or a default parameter value.
+
+    Okay: def complex(real, imag=0.0):
+    Okay: return magic(r=real, i=imag)
+    Okay: boolean(a == b)
+    Okay: boolean(a != b)
+    Okay: boolean(a <= b)
+    Okay: boolean(a >= b)
+
+    E251: def complex(real, imag = 0.0):
+    E251: return magic(r = real, i = imag)
+    """
+    parens = 0
+    window = '   '
+    equal_ok = ['==', '!=', '<=', '>=']
+
+    for pos, c in enumerate(logical_line):
+        window = window[1:] + c
+        if parens:
+            if window[0] in WHITESPACE and window[1] == '=':
+                if window[1:] not in equal_ok:
+                    issue = "E251 no spaces around keyword / parameter equals"
+                    return pos, issue
+            if window[2] in WHITESPACE and window[1] == '=':
+                if window[:2] not in equal_ok:
+                    issue = "E251 no spaces around keyword / parameter equals"
+                    return pos, issue
+        if c == '(':
+            parens += 1
+        elif c == ')':
+            parens -= 1
+
+
+def whitespace_before_inline_comment(logical_line, tokens):
+    """
+    Separate inline comments by at least two spaces.
+
+    An inline comment is a comment on the same line as a statement.  Inline
+    comments should be separated by at least two spaces from the statement.
+    They should start with a # and a single space.
+
+    Okay: x = x + 1  # Increment x
+    Okay: x = x + 1    # Increment x
+    E261: x = x + 1 # Increment x
+    E262: x = x + 1  #Increment x
+    E262: x = x + 1  #  Increment x
+    """
+    prev_end = (0, 0)
+    for token_type, text, start, end, line in tokens:
+        if token_type == tokenize.NL:
+            continue
+        if token_type == tokenize.COMMENT:
+            if not line[:start[1]].strip():
+                continue
+            if prev_end[0] == start[0] and start[1] < prev_end[1] + 2:
+                return (prev_end,
+                        "E261 at least two spaces before inline comment")
+            if (len(text) > 1 and text.startswith('#  ')
+                           or not text.startswith('# ')):
+                return start, "E262 inline comment should start with '# '"
+        else:
+            prev_end = end
+
+
+def imports_on_separate_lines(logical_line):
+    r"""
+    Imports should usually be on separate lines.
+
+    Okay: import os\nimport sys
+    E401: import sys, os
+
+    Okay: from subprocess import Popen, PIPE
+    Okay: from myclas import MyClass
+    Okay: from foo.bar.yourclass import YourClass
+    Okay: import myclass
+    Okay: import foo.bar.yourclass
+    """
+    line = logical_line
+    if line.startswith('import '):
+        found = line.find(',')
+        if found > -1:
+            return found, "E401 multiple imports on one line"
+
+
+def compound_statements(logical_line):
+    r"""
+    Compound statements (multiple statements on the same line) are
+    generally discouraged.
+
+    While sometimes it's okay to put an if/for/while with a small body
+    on the same line, never do this for multi-clause statements. Also
+    avoid folding such long lines!
+
+    Okay: if foo == 'blah':\n    do_blah_thing()
+    Okay: do_one()
+    Okay: do_two()
+    Okay: do_three()
+
+    E701: if foo == 'blah': do_blah_thing()
+    E701: for x in lst: total += x
+    E701: while t < 10: t = delay()
+    E701: if foo == 'blah': do_blah_thing()
+    E701: else: do_non_blah_thing()
+    E701: try: something()
+    E701: finally: cleanup()
+    E701: if foo == 'blah': one(); two(); three()
+
+    E702: do_one(); do_two(); do_three()
+    """
+    line = logical_line
+    found = line.find(':')
+    if -1 < found < len(line) - 1:
+        before = line[:found]
+        if (before.count('{') <= before.count('}') and  # {'a': 1} (dict)
+            before.count('[') <= before.count(']') and  # [1:2] (slice)
+            not re.search(r'\blambda\b', before)):      # lambda x: x
+            return found, "E701 multiple statements on one line (colon)"
+    found = line.find(';')
+    if -1 < found:
+        return found, "E702 multiple statements on one line (semicolon)"
+
+
+def python_3000_has_key(logical_line):
+    """
+    The {}.has_key() method will be removed in the future version of
+    Python. Use the 'in' operation instead, like:
+    d = {"a": 1, "b": 2}
+    if "b" in d:
+        print d["b"]
+    """
+    pos = logical_line.find('.has_key(')
+    if pos > -1:
+        return pos, "W601 .has_key() is deprecated, use 'in'"
+
+
+def python_3000_raise_comma(logical_line):
+    """
+    When raising an exception, use "raise ValueError('message')"
+    instead of the older form "raise ValueError, 'message'".
+
+    The paren-using form is preferred because when the exception arguments
+    are long or include string formatting, you don't need to use line
+    continuation characters thanks to the containing parentheses.  The older
+    form will be removed in Python 3000.
+    """
+    match = RAISE_COMMA_REGEX.match(logical_line)
+    if match:
+        return match.start(1), "W602 deprecated form of raising exception"
+
+
+def python_3000_not_equal(logical_line):
+    """
+    != can also be written <>, but this is an obsolete usage kept for
+    backwards compatibility only. New code should always use !=.
+    The older syntax is removed in Python 3000.
+    """
+    pos = logical_line.find('<>')
+    if pos > -1:
+        return pos, "W603 '<>' is deprecated, use '!='"
+
+
+def python_3000_backticks(logical_line):
+    """
+    Backticks are removed in Python 3000.
+    Use repr() instead.
+    """
+    pos = logical_line.find('`')
+    if pos > -1:
+        return pos, "W604 backticks are deprecated, use 'repr()'"
+
+
+##############################################################################
+# Helper functions
+##############################################################################
+
+
+def expand_indent(line):
+    """
+    Return the amount of indentation.
+    Tabs are expanded to the next multiple of 8.
+
+    >>> expand_indent('    ')
+    4
+    >>> expand_indent('\\t')
+    8
+    >>> expand_indent('    \\t')
+    8
+    >>> expand_indent('       \\t')
+    8
+    >>> expand_indent('        \\t')
+    16
+    """
+    result = 0
+    for char in line:
+        if char == '\t':
+            result = result // 8 * 8 + 8
+        elif char == ' ':
+            result += 1
+        else:
+            break
+    return result
+
+
+def mute_string(text):
+    """
+    Replace contents with 'xxx' to prevent syntax matching.
+
+    >>> mute_string('"abc"')
+    '"xxx"'
+    >>> mute_string("'''abc'''")
+    "'''xxx'''"
+    >>> mute_string("r'abc'")
+    "r'xxx'"
+    """
+    start = 1
+    end = len(text) - 1
+    # String modifiers (e.g. u or r)
+    if text.endswith('"'):
+        start += text.index('"')
+    elif text.endswith("'"):
+        start += text.index("'")
+    # Triple quotes
+    if text.endswith('"""') or text.endswith("'''"):
+        start += 2
+        end -= 2
+    return text[:start] + 'x' * (end - start) + text[end:]
+
+
+def message(text):
+    """Print a message."""
+    # print >> sys.stderr, options.prog + ': ' + text
+    # print >> sys.stderr, text
+    print(text)
+
+
+##############################################################################
+# Framework to run all checks
+##############################################################################
+
+
+def find_checks(argument_name):
+    """
+    Find all globally visible functions where the first argument name
+    starts with argument_name.
+    """
+    checks = []
+    for name, function in globals().items():
+        if not inspect.isfunction(function):
+            continue
+        args = inspect.getargspec(function)[0]
+        if args and args[0].startswith(argument_name):
+            codes = ERRORCODE_REGEX.findall(inspect.getdoc(function) or '')
+            for code in codes or ['']:
+                if not code or not ignore_code(code):
+                    checks.append((name, function, args))
+                    break
+    checks.sort()
+    return checks
+
+
+class Checker(object):
+    """
+    Load a Python source file, tokenize it, check coding style.
+    """
+
+    def __init__(self, filename):
+        if filename:
+            self.filename = filename
+            try:
+                self.lines = open(filename).readlines()
+            except UnicodeDecodeError:
+                # Errors may occur with non-UTF8 files in Python 3000
+                self.lines = open(filename, errors='replace').readlines()
+        else:
+            self.filename = 'stdin'
+            self.lines = []
+        options.counters['physical lines'] = \
+            options.counters.get('physical lines', 0) + len(self.lines)
+
+    def readline(self):
+        """
+        Get the next line from the input buffer.
+        """
+        self.line_number += 1
+        if self.line_number > len(self.lines):
+            return ''
+        return self.lines[self.line_number - 1]
+
+    def readline_check_physical(self):
+        """
+        Check and return the next physical line. This method can be
+        used to feed tokenize.generate_tokens.
+        """
+        line = self.readline()
+        if line:
+            self.check_physical(line)
+        return line
+
+    def run_check(self, check, argument_names):
+        """
+        Run a check plugin.
+        """
+        arguments = []
+        for name in argument_names:
+            arguments.append(getattr(self, name))
+        return check(*arguments)
+
+    def check_physical(self, line):
+        """
+        Run all physical checks on a raw input line.
+        """
+        self.physical_line = line
+        if self.indent_char is None and len(line) and line[0] in ' \t':
+            self.indent_char = line[0]
+        for name, check, argument_names in options.physical_checks:
+            result = self.run_check(check, argument_names)
+            if result is not None:
+                offset, text = result
+                self.report_error(self.line_number, offset, text, check)
+
+    def build_tokens_line(self):
+        """
+        Build a logical line from tokens.
+        """
+        self.mapping = []
+        logical = []
+        length = 0
+        previous = None
+        for token in self.tokens:
+            token_type, text = token[0:2]
+            if token_type in (tokenize.COMMENT, tokenize.NL,
+                              tokenize.INDENT, tokenize.DEDENT,
+                              tokenize.NEWLINE):
+                continue
+            if token_type == tokenize.STRING:
+                text = mute_string(text)
+            if previous:
+                end_line, end = previous[3]
+                start_line, start = token[2]
+                if end_line != start_line:  # different row
+                    if self.lines[end_line - 1][end - 1] not in '{[(':
+                        logical.append(' ')
+                        length += 1
+                elif end != start:  # different column
+                    fill = self.lines[end_line - 1][end:start]
+                    logical.append(fill)
+                    length += len(fill)
+            self.mapping.append((length, token))
+            logical.append(text)
+            length += len(text)
+            previous = token
+        self.logical_line = ''.join(logical)
+        assert self.logical_line.lstrip() == self.logical_line
+        assert self.logical_line.rstrip() == self.logical_line
+
+    def check_logical(self):
+        """
+        Build a line from tokens and run all logical checks on it.
+        """
+        options.counters['logical lines'] = \
+            options.counters.get('logical lines', 0) + 1
+        self.build_tokens_line()
+        first_line = self.lines[self.mapping[0][1][2][0] - 1]
+        indent = first_line[:self.mapping[0][1][2][1]]
+        self.previous_indent_level = self.indent_level
+        self.indent_level = expand_indent(indent)
+        if options.verbose >= 2:
+            print(self.logical_line[:80].rstrip())
+        for name, check, argument_names in options.logical_checks:
+            if options.verbose >= 3:
+                print('   ', name)
+            result = self.run_check(check, argument_names)
+            if result is not None:
+                offset, text = result
+                if isinstance(offset, tuple):
+                    original_number, original_offset = offset
+                else:
+                    for token_offset, token in self.mapping:
+                        if offset >= token_offset:
+                            original_number = token[2][0]
+                            original_offset = (token[2][1]
+                                               + offset - token_offset)
+                self.report_error(original_number, original_offset,
+                                  text, check)
+        self.previous_logical = self.logical_line
+
+    def check_all(self):
+        """
+        Run all checks on the input file.
+        """
+        self.file_errors = 0
+        self.line_number = 0
+        self.indent_char = None
+        self.indent_level = 0
+        self.previous_logical = ''
+        self.blank_lines = 0
+        self.blank_lines_before_comment = 0
+        self.tokens = []
+        parens = 0
+        for token in tokenize.generate_tokens(self.readline_check_physical):
+            # print(tokenize.tok_name[token[0]], repr(token))
+            self.tokens.append(token)
+            token_type, text = token[0:2]
+            if token_type == tokenize.OP and text in '([{':
+                parens += 1
+            if token_type == tokenize.OP and text in '}])':
+                parens -= 1
+            if token_type == tokenize.NEWLINE and not parens:
+                self.check_logical()
+                self.blank_lines = 0
+                self.blank_lines_before_comment = 0
+                self.tokens = []
+            if token_type == tokenize.NL and not parens:
+                if len(self.tokens) <= 1:
+                    # The physical line contains only this token.
+                    self.blank_lines += 1
+                self.tokens = []
+            if token_type == tokenize.COMMENT:
+                source_line = token[4]
+                token_start = token[2][1]
+                if source_line[:token_start].strip() == '':
+                    self.blank_lines_before_comment = max(self.blank_lines,
+                        self.blank_lines_before_comment)
+                    self.blank_lines = 0
+                if text.endswith('\n') and not parens:
+                    # The comment also ends a physical line.  This works around
+                    # Python < 2.6 behaviour, which does not generate NL after
+                    # a comment which is on a line by itself.
+                    self.tokens = []
+        return self.file_errors
+
+    def report_error(self, line_number, offset, text, check):
+        """
+        Report an error, according to options.
+        """
+        if options.quiet == 1 and not self.file_errors:
+            message(self.filename)
+        self.file_errors += 1
+        code = text[:4]
+        options.counters[code] = options.counters.get(code, 0) + 1
+        options.messages[code] = text[5:]
+        if options.quiet:
+            return
+        if options.testsuite:
+            basename = os.path.basename(self.filename)
+            if basename[:4] != code:
+                return  # Don't care about other errors or warnings
+            if 'not' not in basename:
+                return  # Don't print the expected error message
+        if ignore_code(code):
+            return
+        if options.counters[code] == 1 or options.repeat:
+            message("%s:%s:%d: %s" %
+                    (self.filename, line_number, offset + 1, text))
+            if options.show_source:
+                line = self.lines[line_number - 1]
+                message(line.rstrip())
+                message(' ' * offset + '^')
+            if options.show_pep8:
+                message(check.__doc__.lstrip('\n').rstrip())
+
+
+def input_file(filename):
+    """
+    Run all checks on a Python source file.
+    """
+    if excluded(filename):
+        return {}
+    if options.verbose:
+        message('checking ' + filename)
+    files_counter_before = options.counters.get('files', 0)
+    if options.testsuite:  # Keep showing errors for multiple tests
+        options.counters = {}
+    options.counters['files'] = files_counter_before + 1
+    errors = Checker(filename).check_all()
+    if options.testsuite:  # Check if the expected error was found
+        basename = os.path.basename(filename)
+        code = basename[:4]
+        count = options.counters.get(code, 0)
+        if count == 0 and 'not' not in basename:
+            message("%s: error %s not found" % (filename, code))
+
+
+def input_dir(dirname):
+    """
+    Check all Python source files in this directory and all subdirectories.
+    """
+    dirname = dirname.rstrip('/')
+    if excluded(dirname):
+        return
+    for root, dirs, files in os.walk(dirname):
+        if options.verbose:
+            message('directory ' + root)
+        options.counters['directories'] = \
+            options.counters.get('directories', 0) + 1
+        dirs.sort()
+        for subdir in dirs:
+            if excluded(subdir):
+                dirs.remove(subdir)
+        files.sort()
+        for filename in files:
+            if filename_match(filename):
+                input_file(os.path.join(root, filename))
+
+
+def excluded(filename):
+    """
+    Check if options.exclude contains a pattern that matches filename.
+    """
+    basename = os.path.basename(filename)
+    for pattern in options.exclude:
+        if fnmatch(basename, pattern):
+            # print basename, 'excluded because it matches', pattern
+            return True
+
+
+def filename_match(filename):
+    """
+    Check if options.filename contains a pattern that matches filename.
+    If options.filename is unspecified, this always returns True.
+    """
+    if not options.filename:
+        return True
+    for pattern in options.filename:
+        if fnmatch(filename, pattern):
+            return True
+
+
+def ignore_code(code):
+    """
+    Check if options.ignore contains a prefix of the error code.
+    If options.select contains a prefix of the error code, do not ignore it.
+    """
+    for select in options.select:
+        if code.startswith(select):
+            return False
+    for ignore in options.ignore:
+        if code.startswith(ignore):
+            return True
+
+
+def get_error_statistics():
+    """Get error statistics."""
+    return get_statistics("E")
+
+
+def get_warning_statistics():
+    """Get warning statistics."""
+    return get_statistics("W")
+
+
+def get_statistics(prefix=''):
+    """
+    Get statistics for message codes that start with the prefix.
+
+    prefix='' matches all errors and warnings
+    prefix='E' matches all errors
+    prefix='W' matches all warnings
+    prefix='E4' matches all errors that have to do with imports
+    """
+    stats = []
+    keys = list(options.messages.keys())
+    keys.sort()
+    for key in keys:
+        if key.startswith(prefix):
+            stats.append('%-7s %s %s' %
+                         (options.counters[key], key, options.messages[key]))
+    return stats
+
+
+def get_count(prefix=''):
+    """Return the total count of errors and warnings."""
+    keys = list(options.messages.keys())
+    count = 0
+    for key in keys:
+        if key.startswith(prefix):
+            count += options.counters[key]
+    return count
+
+
+def print_statistics(prefix=''):
+    """Print overall statistics (number of errors and warnings)."""
+    for line in get_statistics(prefix):
+        print(line)
+
+
+def print_benchmark(elapsed):
+    """
+    Print benchmark numbers.
+    """
+    print('%-7.2f %s' % (elapsed, 'seconds elapsed'))
+    keys = ['directories', 'files',
+            'logical lines', 'physical lines']
+    for key in keys:
+        if key in options.counters:
+            print('%-7d %s per second (%d total)' % (
+                options.counters[key] / elapsed, key,
+                options.counters[key]))
+
+
+def selftest():
+    """
+    Test all check functions with test cases in docstrings.
+    """
+    count_passed = 0
+    count_failed = 0
+    checks = options.physical_checks + options.logical_checks
+    for name, check, argument_names in checks:
+        for line in check.__doc__.splitlines():
+            line = line.lstrip()
+            match = SELFTEST_REGEX.match(line)
+            if match is None:
+                continue
+            code, source = match.groups()
+            checker = Checker(None)
+            for part in source.split(r'\n'):
+                part = part.replace(r'\t', '\t')
+                part = part.replace(r'\s', ' ')
+                checker.lines.append(part + '\n')
+            options.quiet = 2
+            options.counters = {}
+            checker.check_all()
+            error = None
+            if code == 'Okay':
+                if len(options.counters) > 1:
+                    codes = [key for key in options.counters.keys()
+                             if key != 'logical lines']
+                    error = "incorrectly found %s" % ', '.join(codes)
+            elif options.counters.get(code, 0) == 0:
+                error = "failed to find %s" % code
+            if not error:
+                count_passed += 1
+            else:
+                count_failed += 1
+                if len(checker.lines) == 1:
+                    print("pep8.py: %s: %s" %
+                          (error, checker.lines[0].rstrip()))
+                else:
+                    print("pep8.py: %s:" % error)
+                    for line in checker.lines:
+                        print(line.rstrip())
+    if options.verbose:
+        print("%d passed and %d failed." % (count_passed, count_failed))
+        if count_failed:
+            print("Test failed.")
+        else:
+            print("Test passed.")
+
+
+def process_options(arglist=None):
+    """
+    Process options passed either via arglist or via command line args.
+    """
+    global options, args
+    parser = OptionParser(version=__version__,
+                          usage="%prog [options] input ...")
+    parser.add_option('-v', '--verbose', default=0, action='count',
+                      help="print status messages, or debug with -vv")
+    parser.add_option('-q', '--quiet', default=0, action='count',
+                      help="report only file names, or nothing with -qq")
+    parser.add_option('-r', '--repeat', action='store_true',
+                      help="show all occurrences of the same error")
+    parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE,
+                      help="exclude files or directories which match these "
+                        "comma separated patterns (default: %s)" %
+                        DEFAULT_EXCLUDE)
+    parser.add_option('--filename', metavar='patterns', default='*.py',
+                      help="when parsing directories, only check filenames "
+                        "matching these comma separated patterns (default: "
+                        "*.py)")
+    parser.add_option('--select', metavar='errors', default='',
+                      help="select errors and warnings (e.g. E,W6)")
+    parser.add_option('--ignore', metavar='errors', default='',
+                      help="skip errors and warnings (e.g. E4,W)")
+    parser.add_option('--show-source', action='store_true',
+                      help="show source code for each error")
+    parser.add_option('--show-pep8', action='store_true',
+                      help="show text of PEP 8 for each error")
+    parser.add_option('--statistics', action='store_true',
+                      help="count errors and warnings")
+    parser.add_option('--count', action='store_true',
+                      help="print total number of errors and warnings "
+                        "to standard error and set exit code to 1 if "
+                        "total is not null")
+    parser.add_option('--benchmark', action='store_true',
+                      help="measure processing speed")
+    parser.add_option('--testsuite', metavar='dir',
+                      help="run regression tests from dir")
+    parser.add_option('--doctest', action='store_true',
+                      help="run doctest on myself")
+    options, args = parser.parse_args(arglist)
+    if options.testsuite:
+        args.append(options.testsuite)
+    if len(args) == 0 and not options.doctest:
+        parser.error('input not specified')
+    options.prog = os.path.basename(sys.argv[0])
+    options.exclude = options.exclude.split(',')
+    for index in range(len(options.exclude)):
+        options.exclude[index] = options.exclude[index].rstrip('/')
+    if options.filename:
+        options.filename = options.filename.split(',')
+    if options.select:
+        options.select = options.select.split(',')
+    else:
+        options.select = []
+    if options.ignore:
+        options.ignore = options.ignore.split(',')
+    elif options.select:
+        # Ignore all checks which are not explicitly selected
+        options.ignore = ['']
+    elif options.testsuite or options.doctest:
+        # For doctest and testsuite, all checks are required
+        options.ignore = []
+    else:
+        # The default choice: ignore controversial checks
+        options.ignore = DEFAULT_IGNORE
+    options.physical_checks = find_checks('physical_line')
+    options.logical_checks = find_checks('logical_line')
+    options.counters = {}
+    options.messages = {}
+    return options, args
+
+
+def _main():
+    """
+    Parse options and run checks on Python source.
+    """
+    options, args = process_options()
+    if options.doctest:
+        import doctest
+        doctest.testmod(verbose=options.verbose)
+        selftest()
+    start_time = time.time()
+    for path in args:
+        if os.path.isdir(path):
+            input_dir(path)
+        else:
+            input_file(path)
+    elapsed = time.time() - start_time
+    if options.statistics:
+        print_statistics()
+    if options.benchmark:
+        print_benchmark(elapsed)
+    if options.count:
+        count = get_count()
+        if count:
+            sys.stderr.write(str(count) + '\n')
+            sys.exit(1)
+
+
+if __name__ == '__main__':
+    _main()