[Findit] Enable project classifier and component classifier

appengine/findit/model/crash/crash_config.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Findit for crash (ClusterFuzz & Fracas/Chromecrash) configuration."""
 
+import pickle
+import zlib
+
 from google.appengine.ext import ndb
 
 from model.versioned_config import VersionedConfig
 
 
 class CrashConfig(VersionedConfig):
   """Global configuration of settings for processing Chrome crashes."""
   # Fracas-specific parameters.
   # {
   #   "analysis_result_pubsub_topic": "projects/project-name/topics/name",
   #   "supported_platform_list_by_channel": {
   #     "canary": ["win", "mac"],
   #   },
   # }
   fracas = ndb.JsonProperty(indexed=False, default={})
+
+  ################## Settings shared by Fracas/Clusterfuzz. ##################
+  # Project classifier settings:
+  # {
+  #   "host_directories": [
+  #     "src/chrome/browser/resources/",
+  #     "src/chrome/test/data/layout_tests/",
+  #     "src/media/",
+  #     "src/sdch/",
+  #     "src/testing/",
+  #     "src/third_party/WebKit/",
+  #     "src/third_party/",
+  #     "src/tools/",
+  #     "src/"
+  #   ],
+  #   # Where there is no dep_path found, use function and file_path makers to
+  #   # map a Result or StackFrame to a project name.
+  #     "function_marker_to_project_name": {
+  #       "org.chromium": "chromium",
+  #       "com.google.android.apps.chrome": "clank",
+  #       "android.": "android_os",
+  #       "com.android.": "android_os",
+  #   },
+  #   "file_path_marker_to_project_name": {
+  #     ("https___googleplex-android.googlesource."
+  #      "com_a_platform_manifest.git/"): "android_os",
+  #     "googleplex-android/": "android_os",
+  #   },
+  #
+  #   # Number of frames on top to consider when deciding the crashed project.
+  #   "top_n": 4,
+  #
+  #   # The chromium project should always have the highest rank priority (0).
+  #   # This dict assigns rank priorities to non chromium projects.
+  #   "non_chromium_project_rank_priority" = {
+  #     "clank": -1,
+  #     "android_os": -2,
+  #     "android_os_java": -2,
+  #     "src_internal": -3,
+  #     "others": -4,
+  #   }
+  # }
+  project_classifier = ndb.JsonProperty(indexed=False, default={})
+
+  # Component classifier settings:
+  # {
+  #   # Number of frames on top to consider when deciding the crashed
+  #   #component.
+  #   "top_n": 4,
+  #   "path_function_component": [
+  #     (r"src/third_party/WebKit/Source/core/layout", , "Blink>Layout"),
+  #     ...
+  #   ]
+  # }
+  compressed_component_classifier = ndb.BlobProperty(indexed=False,

[stgao, 2016/05/21 00:54:03]

For compress, an alternative is:
ndb.JsonProperty(compressed=True)

[Sharu Jiang, 2016/05/23 23:54:50]

I found out there is a PickleProperty...

+                                                     default=None)
+
+  @property
+  def component_classifier(self):
+    """Decompresses the compressed_component_classifier."""
+    return ({} if self.compressed_component_classifier is None else
+            pickle.loads(zlib.decompress(self.compressed_component_classifier)))
+
+  @property
+  def str_component_classifier(self):
+    """Converts component_classifier to str, since there are re objects in it.
+
+    Note, this is intended to be called only by the CrashConfig HandleGet so as
+    to better display the current configuration.
+    """
+    component_config = self.component_classifier
+    if not component_config:
+      return {}
+
+    # Convert the pattern strings to re compiled objects.
+    path_function_component = []
+    for path, function, component in component_config.get(

[stgao, 2016/05/21 00:54:03]

Personally, it'd avoid this complexity.

I left a comment on how to cache computed pattern in the CrashConfig instance in
original code.

[stgao, 2016/05/21 00:54:03]

This is a little tricky, especially the code for the encoding is in
handlers/crash/crash_config.py

If this is the way to compress and avoid recompiling, is it possible to put the
two piece of code in the same place?

[Sharu Jiang, 2016/05/23 23:54:50]

I can move them to the same place, but it's not necessary now :)

+        'path_function_component', []):
+      path_function_component.append([
+          path.pattern,
+          function.pattern if function else "",
+          component])
+
+    component_config['path_function_component'] = path_function_component
+
+    return component_config