[Predator] Add Clusterfuzz stacktrace parser.

appengine/findit/crash/clusterfuzz_parser.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import re
+
+from crash import callstack_detectors
+from crash.callstack_filters import FilterFramesAfterBlinkGeneratedCode
+from crash.callstack_filters import FilterJavaJreSdkFrames
+from crash.callstack_filters import FilterV8FramesIfV8APIBindingCode
+from crash.callstack_filters import FilterV8FramesIfV8NotInTopFrames
+from crash.callstack_filters import KeepV8FramesIfV8GeneratedJITCrash
+from crash.callstack_filters import KeepTopNFrames
+from crash.flag_manager import Flag
+from crash.flag_manager import FlagManager
+from crash.stacktrace import CallStackBuffer
+from crash.stacktrace import StackFrame
+from crash.stacktrace import StacktraceBuffer
+from crash.stacktrace_parser import StacktraceParser
+from crash.type_enums import CallStackFormatType
+from crash.type_enums import LanguageType
+from crash.type_enums import SanitizerType
+
+
+TOP_FRAME_HAS_NO_SYMBOLS_REGEX = re.compile(
+    r'.*#0 0x[0-9a-f]+  \(<unknown module>\).*')
+SUMMARY_MARKER = 'SUMMARY:'
+JAVA_FATAL_EXCEPTION_REGEX = re.compile('.*FATAL EXCEPTION.*:')
+
+ANDROID_JOB_TYPE_MARKER = 'android'
+DEFAULT_TOP_N_FRAMES = 7
+
+CALLSTACK_FLAG_GROUP = 'callstack_flags'
+STACKTRACE_FLAG_GROUP = 'stacktrace_flags'
+
+
+def GetCallStackDetector(job_type, sanitizer):
+  """Gets ``CallStackDetector`` instance to detect start of new callstack."""

[wrengr, 2016/12/22 23:28:36]

-> "Returns a ``CallStackDetector`` for the particular sanitizer and job type."

[Sharu Jiang, 2016/12/28 20:16:46]

Done.

+
+  if ANDROID_JOB_TYPE_MARKER in job_type:
+    return callstack_detectors.AndroidJobDetector()
+
+  if sanitizer == SanitizerType.SYZYASAN:

[wrengr, 2016/12/22 23:28:36]

fwiw, rather than doing a bunch of equality tests one at a time, you could jump
directly to the right thing by using a dict:

    # at the top level, so we only build it once.
    callstack_detector_classes = {
        SanitizerType.SYZYASAN: callstack_detectors.SyzyasanDetector,
        SanitizerType.THREAD_SANITIZER: callstack_detectors.TsanDetector,
        ...
    }

    # inside this method:
    try:
        return callstack_detector_classes[sanitizer]()
    except KeyError:
        return None

The performance difference isn't a problem yet, but if we start adding a whole
bunch of different CallStackDetectors then it'll be a lot faster.

[Sharu Jiang, 2016/12/28 20:16:46]

Done.

+    return callstack_detectors.SyzyasanDetector()
+
+  if sanitizer == SanitizerType.THREAD_SANITIZER:
+    return callstack_detectors.TsanDetector()
+
+  if sanitizer == SanitizerType.UBSAN:
+    return callstack_detectors.UbsanDetector()
+
+  if sanitizer == SanitizerType.MEMORY_SANITIZER:
+    return callstack_detectors.MsanDetector()
+
+  if sanitizer == SanitizerType.ADDRESS_SANITIZER:
+    return callstack_detectors.AsanDetector()
+
+  return None
+
+
+class ClusterfuzzParser(StacktraceParser):
+
+  def __init__(self):
+    self.flag_manager = FlagManager()
+    self.flag_manager.Register(STACKTRACE_FLAG_GROUP, Flag(
+        'java_main_stack', lambda line:   # pylint: disable=W0108
+            JAVA_FATAL_EXCEPTION_REGEX.match(line)))
+    self.flag_manager.Register(STACKTRACE_FLAG_GROUP, Flag(
+        'after_summary_line', lambda line:   # pylint: disable=W0108
+            SUMMARY_MARKER in line))
+    # This flag is True at the very beginning and will never be changed once it
+    # is set to False.
+    self.flag_manager.Register(STACKTRACE_FLAG_GROUP, Flag(
+        'is_first_stack', lambda line: False, val=True)) # pylint: disable=W0108
+    self.flag_manager.Register(CALLSTACK_FLAG_GROUP, Flag(
+        'top_frame_has_no_symbol', lambda line:  # pylint: disable=W0108
+            TOP_FRAME_HAS_NO_SYMBOLS_REGEX.match(line)))
+
+  def Parse(self, stacktrace_string, deps, job_type,  # pylint: disable=W0221
+            signature=None, top_n_frames=None,
+            crash_address=None):
+    """Parse clusterfuzz stacktrace string into Stacktrace instance."""
+    filters = [FilterJavaJreSdkFrames(),
+               KeepV8FramesIfV8GeneratedJITCrash(),
+               FilterV8FramesIfV8APIBindingCode(crash_address),
+               FilterFramesAfterBlinkGeneratedCode(),
+               FilterV8FramesIfV8NotInTopFrames(),
+               KeepTopNFrames(top_n_frames or DEFAULT_TOP_N_FRAMES)]
+    sanitizer = SanitizerType.GetSanitizerType(job_type, stacktrace_string)
+    stacktrace_buffer = StacktraceBuffer(signature=signature, filters=filters)
+
+    stack_detector = GetCallStackDetector(job_type, sanitizer)
+    if stack_detector is None:
+      logging.error('Cannot find CallStackDetector for crash %s (job type: %s)', 
+                    signature or '', job_type)
+      return None
+
+    # Initial background callstack which is not to be added into Stacktrace.
+    stack_buffer = CallStackBuffer()
+    # Reset both stacktarce and callstack flags.

[wrengr, 2016/12/22 23:28:36]

"stacktarce" -> "stacktrace"

[Sharu Jiang, 2016/12/28 20:16:46]

Done.

+    self.flag_manager.Reset()
+    for line in stacktrace_string.splitlines():
+      # Note, some flags like is_first_stack may be changed inside of stack
+      # detector.
+      is_new_callstack, priority, format_type, language_type, metadata = (
+          stack_detector.IsStartOfNewCallStack(line, flags=self.flag_manager))
+
+      if is_new_callstack:
+        # Reset callstack scope flags.
+        self.flag_manager.Reset(CALLSTACK_FLAG_GROUP)
+        stacktrace_buffer.AddFilteredStack(stack_buffer)
+        stack_buffer = CallStackBuffer(priority=priority,
+                                       format_type=format_type,
+                                       language_type=language_type,
+                                       metadata=metadata)
+      else:
+        frame = StackFrame.Parse(stack_buffer.language_type,
+                                 stack_buffer.format_type, line, deps,
+                                 len(stack_buffer.frames))
+        if frame is not None:
+          stack_buffer.frames.append(frame)
+        # Turn on flags if condition met.
+        self.flag_manager.ConditionallyTurnOn(line)
+
+    # Add the last stack to stacktrace.
+    stacktrace_buffer.AddFilteredStack(stack_buffer)
+    return stacktrace_buffer.ToStacktrace()