[Findit] scripts to calculate confidence level of Findit results.

appengine/findit/util_scripts/remote_queries/check_confidence_level_try_job.py

Index: appengine/findit/util_scripts/remote_queries/check_confidence_level_try_job.py
diff --git a/appengine/findit/util_scripts/remote_queries/check_confidence_level_try_job.py b/appengine/findit/util_scripts/remote_queries/check_confidence_level_try_job.py
new file mode 100644
index 0000000000000000000000000000000000000000..f6d2504fbe913404879a6eecc7d1183dc8ca5679
--- /dev/null
+++ b/appengine/findit/util_scripts/remote_queries/check_confidence_level_try_job.py
@@ -0,0 +1,83 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Checks confidence level for different result source (try job or heuristic)
+   or different score for heuristic results."""
+from collections import defaultdict
+import datetime
+import json
+import os
+import sys
+
+_REMOTE_API_DIR = os.path.join(os.path.dirname(__file__), os.path.pardir)
+sys.path.insert(1, _REMOTE_API_DIR)
+
+import remote_api
+
+from model import result_status
+from model.wf_analysis import WfAnalysis
+from model.wf_try_job_data import WfTryJobData
+
+
+DAYS = 30
+
+CORRECT_RESULT_STATUSES = [result_status.FOUND_CORRECT,
+                           result_status.FOUND_CORRECT_DUPLICATE]
+INCORRECT_RESULT_STATUSES = [result_status.FOUND_INCORRECT,
+                             result_status.FOUND_INCORRECT_DUPLICATE]
+TRIAGED_RESULT_STATUSES = [result_status.FOUND_CORRECT,
+                           result_status.FOUND_CORRECT_DUPLICATE,
+                           result_status.FOUND_INCORRECT,
+                           result_status.FOUND_INCORRECT_DUPLICATE]
+
+if __name__ == '__main__':
+  # Set up the Remote API to use services on the live App Engine.
+  remote_api.EnableRemoteApi(app_id='findit-for-me')
+
+  end_date = datetime.datetime.utcnow().replace(
+      hour=0, minute=0, second=0, microsecond=0)
+  start_date = end_date - datetime.timedelta(days=DAYS)
+
+  try_job_data_query = WfTryJobData.query(
+      WfTryJobData.request_time >= start_date,
+      WfTryJobData.request_time < end_date)
+  try_job_data_list = try_job_data_query.fetch()
+
+  total_triaged_try_job_num = 0
+  correct_try_job_num = 0
+  total_triaged_compile_num = 0
+  correct_compile_num = 0
+  total_triaged_test_num = 0
+  correct_test_num = 0
+  for data in try_job_data_list:
+    if data.error:
+      continue
+    analysis = WfAnalysis.Get(
+        data.master_name, data.builder_name, data.build_number)
+    if (analysis and analysis.result_status in TRIAGED_RESULT_STATUSES and
+        len(analysis.suspected_cls) == 1):

[stgao, 2016/06/17 18:38:59]

What if the try-job finds nothing while analysis.suspected_cls is from
heuristic?

[chanli, 2016/06/17 22:01:28]

Added a check of data.culprits

+      total_triaged_try_job_num += 1
+      if data.try_job_type == 'compile':

[lijeffrey, 2016/06/17 19:43:11]

just to be safe i'd add data.try_job_type.lower() == 'compile' same for the
other ones

[chanli, 2016/06/17 22:01:28]

Done.

+        total_triaged_compile_num += 1
+      else:
+        total_triaged_test_num += 1
+      if analysis.result_status in CORRECT_RESULT_STATUSES:
+        correct_try_job_num += 1
+        if data.try_job_type == 'compile':
+          correct_compile_num += 1
+        else:
+          correct_test_num += 1
+
+
+  print total_triaged_try_job_num
+  print correct_try_job_num
+  print float(correct_try_job_num)/total_triaged_try_job_num
+  print
+  print total_triaged_compile_num
+  print correct_compile_num
+  print float(correct_compile_num)/total_triaged_compile_num
+  print
+  print total_triaged_test_num
+  print correct_test_num
+  print float(correct_test_num)/total_triaged_test_num