[Findit] Cross-platform triage

appengine/findit/handlers/test/triage_analysis_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from datetime import datetime
+from datetime import timedelta
+
 from google.appengine.ext import ndb
 import webapp2
 
 from testing_utils import testing
 
 from handlers import triage_analysis
+from model import analysis_status
+from model import result_status
 from model.wf_analysis import WfAnalysis
-from model import result_status
-from model import analysis_status
 from waterfall import buildbot
 
 
 class TriageAnalysisTest(testing.AppengineTestCase):
   app_module = webapp2.WSGIApplication([
       ('/triage-analysis', triage_analysis.TriageAnalysis),
   ], debug=True)
 
   def setUp(self):
     super(TriageAnalysisTest, self).setUp()
     self.master_name = 'm'
     self.builder_name = 'b'
     self.build_number_incomplete = 120  # Analysis is not completed yet.
     self.build_number_found = 122  # Suspected CLs are found for this build.
     self.build_number_not_found = 123  # No suspected CLs found.
     self.suspected_cls = [{
         'repo_name': 'chromium',
         'revision': 'r1',
         'commit_position': 123,
         'url': 'https://codereview.chromium.org/123',
     }]
 
+    self.build_start_time = (datetime.utcnow() - timedelta(2)).replace(
+        hour=12, minute=0, second=0, microsecond=0)  # Two days ago, UTC Noon.
+
     analysis = WfAnalysis.Create(
         self.master_name, self.builder_name, self.build_number_incomplete)
     analysis.status = analysis_status.RUNNING
     analysis.put()
 
     analysis = WfAnalysis.Create(
         self.master_name, self.builder_name, self.build_number_found)
     analysis.status = analysis_status.COMPLETED
     analysis.suspected_cls = self.suspected_cls
+    analysis.build_start_time = self.build_start_time
     analysis.put()
 
     analysis = WfAnalysis.Create(
         self.master_name, self.builder_name, self.build_number_not_found)
     analysis.status = analysis_status.COMPLETED
     analysis.put()
 
     self.mock_current_user(user_email='test@chromium.org', is_admin=True)
 
   def testUpdateAnalysisResultStatusWhenAnalysisIsIncomplete(self):
-    success = triage_analysis._UpdateAnalysisResultStatus(
+    success, _ = triage_analysis._UpdateAnalysisResultStatus(
         self.master_name, self.builder_name, self.build_number_incomplete, True)
     self.assertFalse(success)
     analysis = WfAnalysis.Get(
         self.master_name, self.builder_name, self.build_number_found)
     self.assertIsNone(analysis.result_status)
 
   def testUpdateAnalysisResultStatusWhenFoundAndCorrect(self):
     success = triage_analysis._UpdateAnalysisResultStatus(
         self.master_name, self.builder_name, self.build_number_found, True)
     self.assertTrue(success)
@@ skipping 57 matching lines @@
     self.assertEquals({'success': False}, response.json_body)
 
   def testSuccessfulTriage(self):
     build_url = buildbot.CreateBuildUrl(
         self.master_name, self.builder_name, self.build_number_found)
     response = self.test_app.get(
         '/triage-analysis',
         params={'url': build_url, 'correct': True, 'format': 'json'})
     self.assertEquals(200, response.status_int)
     self.assertEquals({'success': True}, response.json_body)
+
+  def testIncompleteTriage(self):
+    build_url = buildbot.CreateBuildUrl(
+        self.master_name, self.builder_name, self.build_number_incomplete)
+    response = self.test_app.get(
+        '/triage-analysis',
+        params={'url': build_url, 'correct': True, 'format': 'json'})
+    self.assertEquals(200, response.status_int)
+    self.assertEquals({'success': False}, response.json_body)
+
+  def testAnalysesMatch(self):
+    analysis_with_empty_failures = WfAnalysis.Create(
+        self.master_name, self.builder_name, 200)
+    analysis_with_empty_failures.result = {
+        'failures': []
+    }
+    analysis_with_empty_failures.put()
+
+    analysis_with_no_suspected_cls = WfAnalysis.Create(
+        self.master_name, self.builder_name, 201)
+    analysis_with_no_suspected_cls.result = {
+        'failures': [
+            {
+                'suspected_cls': []
+            },
+            {
+                'suspected_cls': []
+            },
+        ]
+    }
+    analysis_with_no_suspected_cls.put()
+
+    analysis_with_suspected_cls_1 = WfAnalysis.Create(
+        self.master_name, self.builder_name, 202)
+    analysis_with_suspected_cls_1.result = {
+        'failures': [
+            {
+                'step_name': 'interactive_ui_tests',
+                'suspected_cls': [
+                    {
+                        'revision': '3cf9343f4602d4ec11717cb6ff56a793c1d5f84b',
+                    }
+                ],
+            }
+        ]
+    }
+    analysis_with_suspected_cls_1.put()
+
+    analysis_with_suspected_cls_2 = WfAnalysis.Create(
+        self.master_name, self.builder_name, 203)
+    analysis_with_suspected_cls_2.result = {
+        'failures': [
+            {
+                'suspected_cls': [],
+                'step_name': 'browser_tests'
+            },
+            {
+                'suspected_cls': [
+                    {
+                        'revision': '0e8dc209f5e4a6140e43551de0e036324c68a383',
+                    }
+                ],
+                'step_name': 'content_browsertests'
+            }
+        ]
+    }
+    analysis_with_suspected_cls_2.put()
+
+    analysis_with_suspected_cls_3 = WfAnalysis.Create(
+        self.master_name, self.builder_name, 204)
+    analysis_with_suspected_cls_3.result = {
+        'failures': [
+            {
+                'suspected_cls': [],
+                'step_name': 'browser_tests',
+            },
+            {
+                'suspected_cls': [
+                    {
+                        'revision': '0e8dc209f5e4a6140e43551de0e036324c68a383',
+                    },
+                    {
+                        'revision': '292b41bbd603ae2f11d239f457a8a5f04387fa85',
+                    },
+                    {
+                        'revision': 'f6c9ef029e28a6bef28e727cd70751d782963e21',
+                    }
+                ],
+                'step_name': 'content_browsertests',
+            }
+        ]
+    }
+    analysis_with_suspected_cls_3.result_status = result_status.FOUND_UNTRIAGED
+    analysis_with_suspected_cls_3.build_start_time = self.build_start_time
+    analysis_with_suspected_cls_3.put()
+
+    analysis_with_suspected_cls_4 = WfAnalysis.Create(
+        self.master_name, self.builder_name, 205)
+    analysis_with_suspected_cls_4.result = {
+        'failures': [
+            {
+                'suspected_cls': [],
+                'step_name': 'browser_tests',
+            },
+            {
+                'suspected_cls': [
+                    {
+                        'revision': '0e8dc209f5e4a6140e43551de0e036324c68a383',

[lijeffrey, 2016/06/24 01:30:06]

nit: this is fine but you can get away with using shorter revisions like 'rev1'
'rev2' in general

[josiahk, 2016/06/24 18:12:39]

Ah ok. I changed them. Thanks!

+                    },
+                    {
+                        'revision': '292b41bbd603ae2f11d239f457a8a5f04387fa85',
+                    },
+                    {
+                        'revision': 'f6c9ef029e28a6bef28e727cd70751d782963e21',
+                    }
+                ],
+                'step_name': 'content_browsertests',
+            }
+        ]
+    }
+    analysis_with_suspected_cls_4.result_status = result_status.FOUND_UNTRIAGED
+    analysis_with_suspected_cls_4.build_start_time = self.build_start_time
+    analysis_with_suspected_cls_4.put()
+
+    analysis_with_tests_1 = WfAnalysis.Create(
+        self.master_name, self.builder_name, 206)
+    analysis_with_tests_1.result = {
+        'failures': [
+            {
+                'tests': [
+                    {
+                        'test_name': 'super_test_1',
+                        'suspected_cls': [
+                            {
+                                'revision': 'abc'
+                            }
+                        ]
+                    }, {
+                        'test_name': 'super_test_2',
+                        'suspected_cls': [
+                            {
+                                'revision': 'def'
+                            },
+                            {
+                                'revision': 'ghi'
+                            }
+                        ]
+                    }
+                ],
+                'step_name': 'interactive_ui_tests',
+                'suspected_cls': [
+                    {
+                        'revision': '3cf9343f4602d4ec11717cb6ff56a793c1d5f84b',
+                    }
+                ],
+            }
+        ]
+    }
+    analysis_with_tests_1.put()
+
+    analysis_with_tests_2 = WfAnalysis.Create(
+        self.master_name, self.builder_name, 207)
+    analysis_with_tests_2.result = {
+        'failures': [
+            {
+                'tests': [
+                    {
+                        'test_name': 'super_test_3',
+                        'suspected_cls': [
+                            {
+                                'revision': 'ab'
+                            },
+                            {
+                                'revision': 'cd'
+                            },
+                            {
+                                'revision': 'ef'
+                            }
+                        ]
+                    }
+                ],
+                'step_name': 'interactive_ui_tests',
+                'suspected_cls': [
+                    {
+                        'revision': '3cf9343f4602d4ec11717cb6ff56a793c1d5f84b',
+                    }
+                ],
+            }
+        ]
+    }
+    analysis_with_tests_2.put()
+
+    # Empty failures list.
+    self.assertFalse(triage_analysis._DoAnalysesMatch(
+        analysis_with_empty_failures,
+        analysis_with_empty_failures))
+    # Zero culprit-tuples.
+    self.assertFalse(triage_analysis._DoAnalysesMatch(
+        analysis_with_no_suspected_cls,
+        analysis_with_no_suspected_cls))
+    # Zero culprit-tuples and some culprit-tuples.
+    self.assertFalse(triage_analysis._DoAnalysesMatch(
+        analysis_with_no_suspected_cls,
+        analysis_with_suspected_cls_1))
+    # Has step-level culprit-tuples, and should detect match.
+    self.assertTrue(triage_analysis._DoAnalysesMatch(
+        analysis_with_suspected_cls_2,
+        analysis_with_suspected_cls_2))
+    # Two different step-level culprit-tuples, and should fail to match.
+    self.assertFalse(triage_analysis._DoAnalysesMatch(
+        analysis_with_suspected_cls_2,
+        analysis_with_suspected_cls_3))
+    # Has test-level culprit-tuples, and should detect match.
+    self.assertTrue(triage_analysis._DoAnalysesMatch(
+        analysis_with_tests_1,
+        analysis_with_tests_1))
+    # Two different test-level culprit-tuples, and should fail to match.
+    self.assertFalse(triage_analysis._DoAnalysesMatch(
+        analysis_with_tests_1,
+        analysis_with_tests_2))
+
+  def createAnalysis(self, build_number, build_start_time):

[lijeffrey, 2016/06/24 01:30:06]

nit: I would make this an internal method, i.e. _createAnalysis(...)

[josiahk, 2016/06/24 18:12:39]

Done.

+    analysis = WfAnalysis.Create(
+        self.master_name, self.builder_name, build_number)
+    analysis.result = {
+        'failures': [
+            {
+                'suspected_cls': [
+                    {
+                        'revision': 'abc',
+                    }
+                ],
+                'step_name': 'turing_test',
+            }
+        ]
+    }
+    analysis.result_status = result_status.FOUND_UNTRIAGED
+    analysis.build_start_time = build_start_time
+    analysis.status = analysis_status.COMPLETED
+    analysis.put()
+    return analysis
+
+  def testGetDuplicateAnalysesTooEarly(self):
+    # Two days ago, UTC Noon.
+    original_time = (datetime.utcnow() - timedelta(days=2)).replace(
+        hour=12, minute=0, second=0, microsecond=0)
+    analysis_original = self.createAnalysis(300, original_time)
+
+    # An earlier time, outside bounds.
+    too_early_time = (original_time - timedelta(
+        hours=triage_analysis.MATCHING_ANALYSIS_HOURS_AGO_START*2))
+    self.createAnalysis(301, too_early_time)
+
+    self.assertEquals(
+        len(triage_analysis._GetDuplicateAnalyses(analysis_original)), 0)
+
+  def testGetDuplicateAnalysesEarlier(self):
+    # Two days ago, UTC Noon.
+    original_time = (datetime.utcnow() - timedelta(days=2)).replace(
+        hour=12, minute=0, second=0, microsecond=0)
+    analysis_original = self.createAnalysis(302, original_time)
+
+    # An earlier time, within bounds.
+    earlier_time = (original_time - timedelta(
+        hours=triage_analysis.MATCHING_ANALYSIS_HOURS_AGO_START/2))
+    self.createAnalysis(303, earlier_time)
+
+    self.assertEquals(
+        len(triage_analysis._GetDuplicateAnalyses(analysis_original)), 1)
+
+  def testGetDuplicateAnalysesLater(self):
+    # Two days ago, UTC Noon.
+    original_time = (datetime.utcnow() - timedelta(days=2)).replace(
+        hour=12, minute=0, second=0, microsecond=0)
+    analysis_original = self.createAnalysis(304, original_time)
+
+    # A later time, within bounds.
+    later_time = (original_time + timedelta(
+        hours=triage_analysis.MATCHING_ANALYSIS_HOURS_AGO_START/2))
+    self.createAnalysis(305, later_time)
+
+    self.assertEquals(
+        len(triage_analysis._GetDuplicateAnalyses(analysis_original)), 1)
+
+  def testGetDuplicateAnalysesTooLate(self):
+    # Two days ago, UTC Noon.
+    original_time = (datetime.utcnow() - timedelta(days=2)).replace(
+        hour=12, minute=0, second=0, microsecond=0)
+    analysis_original = self.createAnalysis(306, original_time)
+
+    # A later time, outside bounds.
+    too_late_time = (original_time + timedelta(
+        hours=triage_analysis.MATCHING_ANALYSIS_HOURS_AGO_START*2))
+    self.createAnalysis(307, too_late_time)
+
+    self.assertEquals(
+        len(triage_analysis._GetDuplicateAnalyses(analysis_original)), 0)
+
+  def testGetDuplicateAnalysesNotToday(self):
+    # Tomorrow, UTC Noon.
+    original_time = (datetime.utcnow() + timedelta(days=1)).replace(
+        hour=12, minute=0, second=0, microsecond=0)
+    analysis_original = self.createAnalysis(308, original_time)
+
+    # Create another analysis at the same time (also tomorrow).
+    self.createAnalysis(309, original_time)
+
+    self.assertEquals(
+        len(triage_analysis._GetDuplicateAnalyses(analysis_original)), 0)
+
+  def testTriageDuplicateResults(self):
+    # Two days ago, UTC Noon.
+    original_time = (datetime.utcnow() - timedelta(days=2)).replace(
+        hour=12, minute=0, second=0, microsecond=0)
+    analysis_original = self.createAnalysis(310, original_time)
+
+    # Create another analysis at the same time (also two days ago).
+    self.createAnalysis(311, original_time)
+
+    triage_analysis._TriageDuplicateResults(analysis_original, True)
+
+    second_analysis = WfAnalysis.Get(self.master_name, self.builder_name, 311)
+
+    self.assertEquals(result_status.NOT_FOUND_CORRECT,
+                      second_analysis.result_status)
+