[Findit] Updating flakiness dashboard to be able to triage results

appengine/findit/handlers/flake/list_flakes.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from google.appengine.ext import ndb
 
+from datetime import datetime
+from datetime import time
+from datetime import timedelta
+
 from common.base_handler import BaseHandler
 from common.base_handler import Permission
 from common import time_util
+from model import result_status
 from model.flake.master_flake_analysis import MasterFlakeAnalysis
 
 
-def FilterMasterFlakeAnalysis(master_flake_analysis_query, master_name,
-                              builder_name, build_number, step_name, test_name):
+def FilterMasterFlakeAnalysis(
+    master_flake_analysis_query, master_name=None, builder_name=None,
+    build_number=None, step_name=None, test_name=None, start_date=None,
+    end_date=None, status_code=result_status.UNSPECIFIED):
   if master_name:
     master_flake_analysis_query = master_flake_analysis_query.filter(
         MasterFlakeAnalysis.master_name == master_name)
   if builder_name:
     master_flake_analysis_query = master_flake_analysis_query.filter(
         MasterFlakeAnalysis.builder_name == builder_name)
   if build_number:
     master_flake_analysis_query = master_flake_analysis_query.filter(
         MasterFlakeAnalysis.build_number == build_number)
   if step_name:
     master_flake_analysis_query = master_flake_analysis_query.filter(
         MasterFlakeAnalysis.step_name == step_name)
   if test_name:
     master_flake_analysis_query = master_flake_analysis_query.filter(
         MasterFlakeAnalysis.test_name == test_name)
-  if not (master_name or builder_name or build_number or
-          step_name or test_name):
-    master_flake_analysis_query.order(-MasterFlakeAnalysis.request_time)
+  if start_date:
+    master_flake_analysis_query = master_flake_analysis_query.filter(
+        MasterFlakeAnalysis.request_time >= start_date)
+  if end_date:
+    master_flake_analysis_query = master_flake_analysis_query.filter(
+        MasterFlakeAnalysis.request_time < end_date)
+  if status_code != result_status.UNSPECIFIED:
+    master_flake_analysis_query = master_flake_analysis_query.filter(
+        MasterFlakeAnalysis.result_status == status_code)
+
+  master_flake_analysis_query.order(-MasterFlakeAnalysis.request_time)
   return master_flake_analysis_query.fetch()
 
 
 class ListFlakes(BaseHandler):
   PERMISSION_LEVEL = Permission.ANYONE
 
+  def _GetStartAndEndDates(self, triage):  # pragma: no cover

[chanli, 2016/10/21 05:49:28]

Add tests? Or you just forgot to remove that pragma: no cover?

[lijeffrey, 2016/10/21 20:02:01]

Done.

+    start_date = None
+    end_date = None
+
+    if triage:
+      midnight_today = datetime.combine(time_util.GetUTCNow(), time.min)
+      midnight_yesterday = midnight_today - timedelta(days=1)
+      midnight_tomorrow = midnight_today + timedelta(days=1)
+
+      start = self.request.get('start_date')
+      end = self.request.get('end_date')
+      start_date = (datetime.strptime(start, '%Y-%m-%d') if start else
+                    midnight_yesterday)
+      end_date = (datetime.strptime(end, '%Y-%m-%d') if end else
+                  midnight_tomorrow)

[chanli, 2016/10/21 05:49:28]

Question: it seems that if I have triage=1 and I want to filter all analysis of
a master, I can only get data from yesterday? Of course I can remove triage=1,
but it seems a littler strange...

[lijeffrey, 2016/10/21 20:02:01]

triage=1 only makes it so the date picker shows up, you can also change the
dates in the UI and filter by master too, assuming the update to index.yaml
indeed fixes the indexing exception that happens right now

[chanli, 2016/10/21 22:50:59]

I just checked, so if I add triage=1, I can only get data from yesterday to
tomorrow. I guess what I'm trying to say is we don't need the upper bound and
lower bound when user doesn't specify the dates.

[chanli, 2016/10/24 20:53:27]

Acknowledged.

+
+    return start_date, end_date
+
   def HandleGet(self):
+    status_code = int(self.request.get('result_status', '-1'))

[chanli, 2016/10/21 05:49:28]

Is -1 result_status.UNSPECIFIED?

[lijeffrey, 2016/10/21 20:02:01]

Done.

     master_name = self.request.get('master_name').strip()
     builder_name = self.request.get('builder_name').strip()
     build_number = self.request.get('build_number').strip()
     if build_number:
       build_number = int(build_number)
     step_name = self.request.get('step_name').strip()
     test_name = self.request.get('test_name').strip()
+    triage = self.request.get('triage') == '1'
+
+    # Only allow querying by start/end dates for admins during triage to avoid
+    # overcomplicating the UI for other users.
+    start_date, end_date = self._GetStartAndEndDates(triage)
 
     master_flake_analyses = FilterMasterFlakeAnalysis(
         MasterFlakeAnalysis.query(), master_name, builder_name, build_number,
-        step_name, test_name)
-    data = {'master_flake_analyses': []}
+        step_name, test_name, start_date, end_date, status_code)
+
+    data = {
+        'master_flake_analyses': [],
+        'result_status_filter': status_code,
+        'master_name_filter': master_name,
+        'builder_name_filter': builder_name,
+        'build_number_filter': build_number,
+        'step_name_filter': step_name,
+        'test_name_filter': test_name
+    }
+
+    if triage:  # pragma: no cover
+      data['triage'] = triage
+      data['start_date'] = start_date
+      data['end_date'] = end_date
 
     for master_flake_analysis in master_flake_analyses:
       data['master_flake_analyses'].append({
           'master_name': master_flake_analysis.master_name,
           'builder_name': master_flake_analysis.builder_name,
           'build_number': master_flake_analysis.build_number,
           'step_name': master_flake_analysis.step_name,
           'test_name': master_flake_analysis.test_name,
           'status': master_flake_analysis.status_description,
           'suspected_build': master_flake_analysis.suspected_flake_build_number,
           'request_time': time_util.FormatDatetime(
               master_flake_analysis.request_time),
+          'result_status': result_status.RESULT_STATUS_TO_DESCRIPTION.get(
+              master_flake_analysis.result_status)
       })
 
     # TODO (stgao): use index instead of in-memory sort.
     # Index doesn't work for now, possibly due to legacy data.
     data['master_flake_analyses'].sort(
-        key=lambda e : e['request_time'], reverse=True)
+        key=lambda e: e['request_time'], reverse=True)
 
     return {
         'template': 'flake/dashboard.html',
         'data': data
     }