Port cpplint's C++11 raw string support to blink's cpp style checker.

third_party/WebKit/Tools/Scripts/webkitpy/style/checkers/cpp.py

 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2009, 2010, 2012 Google Inc. All rights reserved.
 # Copyright (C) 2009 Torch Mobile Inc.
 # Copyright (C) 2009 Apple Inc. All rights reserved.
 # Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org)
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
@@ skipping 671 matching lines @@
 
     Returns:
       True, if next character appended to 'line' is inside a
       string constant.
     """
 
     line = line.replace(r'\\', 'XX')  # after this, \\" does not match to \"
     return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1
 
 
+def cleanse_raw_strings(raw_lines):
+    """Removes C++11 raw strings from lines.
+
+      Before:
+        static const char kData[] = R"(
+            multi-line string
+            )";
+
+      After:
+        static const char kData[] = ""
+            (replaced by blank line)
+            "";
+
+    Args:
+      raw_lines: list of raw lines.
+
+    Returns:
+      list of lines with C++11 raw strings replaced by empty strings.
+    """
+
+    delimiter = None
+    lines_without_raw_strings = []
+    for line in raw_lines:
+        if delimiter:
+            # Inside a raw string, look for the end
+            end = line.find(delimiter)
+            if end >= 0:
+                # Found the end of the string, match leading space for this
+                # line and resume copying the original lines, and also insert
+                # a "" on the last line.
+                leading_space = match(r'^(\s*)\S', line)
+                line = leading_space.group(1) + '""' + line[end + len(delimiter):]
+                delimiter = None
+            else:
+                # Haven't found the end yet, append a blank line.
+                line = '""'
+
+        # Look for beginning of a raw string, and replace them with
+        # empty strings.  This is done in a loop to handle multiple raw
+        # strings on the same line.
+        while delimiter is None:
+            # Look for beginning of a raw string.
+            # See 2.14.15 [lex.string] for syntax.
+            #
+            # Once we have matched a raw string, we check the prefix of the
+            # line to make sure that the line is not part of a single line
+            # comment.  It's done this way because we remove raw strings
+            # before removing comments as opposed to removing comments
+            # before removing raw strings.  This is because there are some
+            # cpplint checks that requires the comments to be preserved, but
+            # we don't want to check comments that are inside raw strings.
+            matched = match(r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line)
+            if matched and not match(r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//', matched.group(1)):
+                delimiter = ')' + matched.group(2) + '"'
+
+                end = matched.group(3).find(delimiter)
+                if end >= 0:
+                    # Raw string ended on same line
+                    line = (matched.group(1) + '""' +
+                            matched.group(3)[end + len(delimiter):])
+                    delimiter = None
+                else:
+                    # Start of a multi-line raw string
+                    line = matched.group(1) + '""'
+            else:
+                break
+
+        lines_without_raw_strings.append(line)
+
+    # TODO(unknown): if delimiter is not None here, we might want to
+    # emit a warning for unterminated string.
+    return lines_without_raw_strings
+
+
 def find_next_multi_line_comment_start(lines, line_index):
     """Find the beginning marker for a multiline comment."""
     while line_index < len(lines):
         if lines[line_index].strip().startswith('/*'):
             # Only return this marker if the comment goes beyond this line
             if lines[line_index].strip().find('*/', 2) < 0:
                 return line_index
         line_index += 1
     return len(lines)
 
@@ skipping 41 matching lines @@
       The line with single-line comments removed.
     """
     comment_position = line.find('//')
     if comment_position != -1 and not is_cpp_string(line[:comment_position]):
         line = line[:comment_position]
     # get rid of /* ... */
     return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line)
 
 
 class CleansedLines(object):
-    """Holds 3 copies of all lines with different preprocessing applied to them.
+    """Holds 4 copies of all lines with different preprocessing applied to them.
 
-    1) elided member contains lines without strings and comments,
-    2) lines member contains lines without comments, and
-    3) raw member contains all the lines without processing.
-    All these three members are of <type 'list'>, and of the same length.
+    1) elided member contains lines without strings and comments.
+    2) lines member contains lines without comments.
+    3) raw_lines member contains all the lines without processing.
+    4) lines_without_raw_strings member is same as raw_lines, but with C++11 raw
+       strings removed.
+    All these members are of <type 'list'>, and of the same length.
     """
 
     def __init__(self, lines):
         self.elided = []
         self.lines = []
         self.raw_lines = lines
         self._num_lines = len(lines)
-        for line_number in range(len(lines)):
-            self.lines.append(cleanse_comments(lines[line_number]))
-            elided = self.collapse_strings(lines[line_number])
+        self.lines_without_raw_strings = cleanse_raw_strings(lines)
+        for line_number in range(len(self.lines_without_raw_strings)):
+            self.lines.append(cleanse_comments(self.lines_without_raw_strings[line_number]))
+            elided = self.collapse_strings(self.lines_without_raw_strings[line_number])
             self.elided.append(cleanse_comments(elided))
 
     def num_lines(self):
         """Returns the number of lines represented."""
         return self._num_lines
 
     @staticmethod
     def collapse_strings(elided):
         """Collapses strings and chars on a line to simple "" or '' blocks.
 
@@ skipping 130 matching lines @@
 
     # If it's a full path and starts with Source/, replace Source with blink
     # since that will be the new style directory.
     filename = sub(r'^Source\/', 'blink/', filename)
 
     standard_name = sub(r'[-.\s\/]', '_', filename).upper() + '_'
 
     return standard_name
 
 
-def check_for_header_guard(filename, lines, error):
+def check_for_header_guard(filename, clean_lines, error):
     """Checks that the file contains a header guard.
 
     Logs an error if no #ifndef header guard is present.  For other
     headers, checks that the full pathname is used.
 
     Args:
       filename: The name of the C++ header file.
       lines: An array of strings, each representing a line of the file.
       error: The function to call with any errors found.
     """
+    raw_lines = clean_lines.lines_without_raw_strings
 
     legacy_cpp_var = get_legacy_header_guard_cpp_variable(filename)
     cpp_var = get_header_guard_cpp_variable(filename)
 
     ifndef = None
     ifndef_line_number = 0
     define = None
-    for line_number, line in enumerate(lines):
+    for line_number, line in enumerate(raw_lines):
         line_split = line.split()
         if len(line_split) >= 2:
             # find the first occurrence of #ifndef and #define, save arg
             if not ifndef and line_split[0] == '#ifndef':
                 # set ifndef to the header guard presented on the #ifndef line.
                 ifndef = line_split[1]
                 ifndef_line_number = line_number
             if not define and line_split[0] == '#define':
                 define = line_split[1]
             if define and ifndef:
@@ skipping 749 matching lines @@
     line, don't end a function with a blank line, don't have too many
     blank lines in a row.
 
     Args:
       file_extension: The current file extension, without the leading dot.
       clean_lines: A CleansedLines instance containing the file.
       line_number: The number of the line to check.
       error: The function to call with any errors found.
     """
 
-    line = clean_lines.elided[line_number]  # get rid of comments and strings
+    # Don't use "elided" lines here, otherwise we can't check commented lines.
+    # Don't want to use "raw" either, because we don't want to check inside C++11
+    # raw strings,
+    raw = clean_lines.lines_without_raw_strings
+    line = raw[line_number]
 
     # You shouldn't have a space before a semicolon at the end of the line.
     # There's a special case for "for" since the style guide allows space before
     # the semicolon there.
     if search(r':\s*;\s*$', line):
         error(line_number, 'whitespace/semicolon', 5,
               'Semicolon defining empty statement. Use { } instead.')
     elif search(r'^\s*;\s*$', line):
         error(line_number, 'whitespace/semicolon', 5,
               'Line contains only semicolon. If this should be an empty statement, '
@@ skipping 626 matching lines @@
       line_number: The number of the line to check.
       file_extension: The extension (without the dot) of the filename.
       class_state: A _ClassState instance which maintains information about
                    the current stack of nested class declarations being parsed.
       file_state: A _FileState instance which maintains information about
                   the state of things in the file.
       enum_state: A _EnumState instance which maintains the current enum state.
       error: The function to call with any errors found.
     """
 
-    raw_lines = clean_lines.raw_lines
+    # Don't use "elided" lines here, otherwise we can't check commented lines.
+    # Don't want to use "raw" either, because we don't want to check inside C++11
+    # raw strings,
+    raw_lines = clean_lines.lines_without_raw_strings
     line = raw_lines[line_number]
 
     # Some more style checks
     check_using_std(clean_lines, line_number, file_state, error)
     check_max_min_macros(clean_lines, line_number, file_state, error)
     check_ctype_functions(clean_lines, line_number, file_state, error)
     check_braces(clean_lines, line_number, error)
     check_exit_statement_simplifications(clean_lines, line_number, error)
     check_spacing(file_extension, clean_lines, line_number, error)
     check_check(clean_lines, line_number, error)
@@ skipping 906 matching lines @@
       error: A callable to which errors are reported, which takes 4 arguments:
     """
     lines = (['// marker so line numbers and indices both start at 1'] + lines +
              ['// marker so line numbers end in a known way'])
 
     include_state = _IncludeState()
     function_state = _FunctionState(min_confidence)
     class_state = _ClassState()
 
     check_for_copyright(lines, error)
+    remove_multi_line_comments(lines, error)
+    clean_lines = CleansedLines(lines)
 
     if file_extension == 'h':
-        check_for_header_guard(filename, lines, error)
+        check_for_header_guard(filename, clean_lines, error)
 
-    remove_multi_line_comments(lines, error)
-    clean_lines = CleansedLines(lines)
     file_state = _FileState(clean_lines, file_extension)
     enum_state = _EnumState()
     for line in xrange(clean_lines.num_lines()):
         process_line(filename, file_extension, clean_lines, line,
                      include_state, function_state, class_state, file_state,
                      enum_state, error)
     class_state.check_finished(error)
 
     check_for_include_what_you_use(filename, clean_lines, include_state, error)
 
@@ skipping 113 matching lines @@
 
     def check(self, lines):
         _process_lines(self.file_path, self.file_extension, lines,
                        self.handle_style_error, self.min_confidence)
 
 
 # FIXME: Remove this function (requires refactoring unit tests).
 def process_file_data(filename, file_extension, lines, error, min_confidence, fs=None):
     checker = CppChecker(filename, file_extension, error, min_confidence, fs)
     checker.check(lines)