[Predator] Switch from scorer-based classifier to loglinear classifier.

appengine/findit/crash/findit_for_chromecrash.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 
 from google.appengine.ext import ndb
 
 from common import appengine_util
 from crash import detect_regression_range
-from crash.changelist_classifier import ChangelistClassifier
 from crash.chromecrash_parser import ChromeCrashParser
 from crash.component import Component
 from crash.component_classifier import ComponentClassifier
 from crash.findit import Findit
+from crash.loglinear.changelist_classifier import LogLinearChangelistClassifier
+from crash.loglinear.changelist_features.touch_crashed_file_meta import (
+    TouchCrashedFileMetaFeature)
+from crash.loglinear.feature import WrapperMetaFeature
+from crash.loglinear.weight import MetaWeight
+from crash.loglinear.weight import Weight
 from crash.predator import Predator
 from crash.project import Project
 from crash.project_classifier import ProjectClassifier
 from crash.type_enums import CrashClient
 from model.crash.cracas_crash_analysis import CracasCrashAnalysis
 from model.crash.crash_config import CrashConfig
 from model.crash.fracas_crash_analysis import FracasCrashAnalysis
 
 # TODO(katesonia): Remove the default value after adding validity check to
 # config.
 _FRACAS_FEEDBACK_URL_TEMPLATE = 'https://%s/crash/fracas-result-feedback?key=%s'
 
 # TODO(wrengr): [Note#1] in many places below we have to do some ugly
 # defaulting in case crash_data is missing certain keys. If we had
 # crash_data be a proper class, rather than an anonymous dict, then we
 # could clean all this up by having the properties themselves do the check
 # and return the default whenever keys are missing. This would also
 # let us do things like have regression_range be automatically computed
 # from historical_metadata (when historical_metadata is provided and
 # regression_range is not).
 
+
 class FinditForChromeCrash(Findit):
   """Find culprits for crash reports from the Chrome Crash server."""
 
   @classmethod
   def _ClientID(cls): # pragma: no cover
     if cls is FinditForChromeCrash:
       logging.warning('FinditForChromeCrash is abstract, '
           'but someone constructed an instance and called _ClientID')
     else:
       logging.warning(
           'FinditForChromeCrash subclass %s forgot to implement _ClientID',
           cls.__name__)
     raise NotImplementedError()
 
   # TODO(http://crbug.com/659354): remove the dependency on CrashConfig
   # entirely, by passing the relevant data as arguments to this constructor.
   def __init__(self, get_repository):
     super(FinditForChromeCrash, self).__init__(get_repository)
+
+    meta_weight = MetaWeight({

[wrengr, 2017/01/31 19:42:11]

This is fine for now, but eventually we'll want to take the weights in as an
argument so that we can read them from a config somewhere rather than hardcoding
them

[Sharu Jiang, 2017/02/01 19:32:40]

Acknowledged. Added a todo.

+        'TouchCrashedFileMeta': MetaWeight({
+            'MinDistance': Weight(1.),
+            'TopFrameIndex': Weight(1.),
+            'TouchCrashedFile': Weight(1.),
+        })
+    })
+    meta_feature = WrapperMetaFeature(
+        [TouchCrashedFileMetaFeature(get_repository)])
+
     project_classifier_config = CrashConfig.Get().project_classifier
-    component_classifier_config = CrashConfig.Get().component_classifier
-
-    self._stacktrace_parser = ChromeCrashParser()
-
     projects = [Project(name, path_regexs, function_regexs, host_directories)
                 for name, path_regexs, function_regexs, host_directories
                 in project_classifier_config['project_path_function_hosts']]
+    component_classifier_config = CrashConfig.Get().component_classifier
     components = [Component(component_name, path_regex, function_regex)
                   for path_regex, function_regex, component_name
-                  in component_classifier_config['path_function_component']],
-    # The top_n is the number of components we should return as
-    # components suggestion results.
+                  in component_classifier_config['path_function_component']]
+    # The top_n is the number of frames we want to check to get component or
+    # project classifications.
     # TODO(http://crbug.com/679964) Deprecate the scorer-based changelist

[wrengr, 2017/01/31 19:42:11]

This todo is redundant now

[Sharu Jiang, 2017/02/01 19:32:39]

Done.

     # classifier and use loglinear model instead.
     self._predator = Predator(
-        cl_classifier = ChangelistClassifier(get_repository),
+        cl_classifier = LogLinearChangelistClassifier(get_repository,
+                                                      meta_feature,
+                                                      meta_weight),
         component_classifier = ComponentClassifier(
             components, component_classifier_config['top_n']),
         project_classifier = ProjectClassifier(
             projects, project_classifier_config['top_n'],
             project_classifier_config['non_chromium_project_rank_priority']))
 
+    self._stacktrace_parser = ChromeCrashParser()
+
   def _InitializeAnalysis(self, model, crash_data):
     super(FinditForChromeCrash, self)._InitializeAnalysis(model, crash_data)
     # TODO(wrengr): see Note#1
     customized_data = crash_data.get('customized_data', {})
     model.channel = customized_data.get('channel', None)
     model.historical_metadata = customized_data.get('historical_metadata', [])
 
   # TODO(wrengr): see Note#1, which would allow us to lift this
   # implementation to the Findit base class.
   @ndb.transactional
@@ skipping 88 matching lines @@
 
   def GetAnalysis(self, crash_identifiers):
     # TODO: inline FracasCrashAnalysis.Get stuff here.
     return FracasCrashAnalysis.Get(crash_identifiers)
 
   def ProcessResultForPublishing(self, result, key):
     """Fracas specific processing of result data for publishing."""
     result['feedback_url'] = _FRACAS_FEEDBACK_URL_TEMPLATE % (
         appengine_util.GetDefaultVersionHostname(), key)
     return result