Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1182)

Side by Side Diff: appengine/findit/waterfall/test/identify_try_job_culprit_pipeline_test.py

Issue 2230103002: [Findit] Pipeline change to save suspected cls to data store. (Closed) Base URL: https://chromium.googlesource.com/infra/infra.git@0808-resubmit-suspected_cl_model
Patch Set: rebase Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2015 The Chromium Authors. All rights reserved. 1 # Copyright 2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 from testing_utils import testing 5 from testing_utils import testing
6 6
7 from common.git_repository import GitRepository 7 from common.git_repository import GitRepository
8 from common.waterfall import failure_type 8 from common.waterfall import failure_type
9 from model import analysis_approach_type
9 from model import analysis_status 10 from model import analysis_status
10 from model import result_status 11 from model import result_status
12 from model.base_build_model import BaseBuildModel
11 from model.wf_analysis import WfAnalysis 13 from model.wf_analysis import WfAnalysis
14 from model.wf_suspected_cl import WfSuspectedCL
12 from model.wf_try_job import WfTryJob 15 from model.wf_try_job import WfTryJob
13 from model.wf_try_job_data import WfTryJobData 16 from model.wf_try_job_data import WfTryJobData
14 from waterfall import identify_try_job_culprit_pipeline 17 from waterfall import identify_try_job_culprit_pipeline
15 from waterfall.identify_try_job_culprit_pipeline import( 18 from waterfall.identify_try_job_culprit_pipeline import(
16 IdentifyTryJobCulpritPipeline) 19 IdentifyTryJobCulpritPipeline)
17 20
18 21
19 class IdentifyTryJobCulpritPipelineTest(testing.AppengineTestCase): 22 class IdentifyTryJobCulpritPipelineTest(testing.AppengineTestCase):
20 23
21 def _MockGetChangeLog(self, revision): 24 def _MockGetChangeLog(self, revision):
22 class MockedChangeLog(object): 25 class MockedChangeLog(object):
23 26
24 def __init__(self, commit_position, code_review_url): 27 def __init__(self, commit_position, code_review_url):
25 self.commit_position = commit_position 28 self.commit_position = commit_position
26 self.code_review_url = code_review_url 29 self.code_review_url = code_review_url
27 30
28 mock_change_logs = {} 31 mock_change_logs = {}
29 mock_change_logs['rev1'] = MockedChangeLog('1', 'url_1') 32 mock_change_logs['rev1'] = MockedChangeLog(1, 'url_1')
30 mock_change_logs['rev2'] = MockedChangeLog('2', 'url_2') 33 mock_change_logs['rev2'] = MockedChangeLog(2, 'url_2')
31 return mock_change_logs.get(revision) 34 return mock_change_logs.get(revision)
32 35
33 def setUp(self): 36 def setUp(self):
34 super(IdentifyTryJobCulpritPipelineTest, self).setUp() 37 super(IdentifyTryJobCulpritPipelineTest, self).setUp()
35 38
36 self.mock(GitRepository, 'GetChangeLog', self._MockGetChangeLog) 39 self.mock(GitRepository, 'GetChangeLog', self._MockGetChangeLog)
37 40
38 def testGetFailedRevisionFromResultsDict(self): 41 def testGetFailedRevisionFromResultsDict(self):
39 self.assertIsNone( 42 self.assertIsNone(
40 identify_try_job_culprit_pipeline._GetFailedRevisionFromResultsDict({})) 43 identify_try_job_culprit_pipeline._GetFailedRevisionFromResultsDict({}))
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
94 def testGetResultAnalysisStatusWithTryJobCulpritNotFoundUntriaged(self): 97 def testGetResultAnalysisStatusWithTryJobCulpritNotFoundUntriaged(self):
95 # Heuristic analysis provided no results, but the try job found a culprit. 98 # Heuristic analysis provided no results, but the try job found a culprit.
96 analysis = WfAnalysis.Create('m', 'b', 1) 99 analysis = WfAnalysis.Create('m', 'b', 1)
97 analysis.result_status = result_status.NOT_FOUND_UNTRIAGED 100 analysis.result_status = result_status.NOT_FOUND_UNTRIAGED
98 analysis.put() 101 analysis.put()
99 102
100 result = { 103 result = {
101 'culprit': { 104 'culprit': {
102 'compile': { 105 'compile': {
103 'revision': 'rev1', 106 'revision': 'rev1',
104 'commit_position': '1', 107 'commit_position': 1,
105 'url': 'url_1', 108 'url': 'url_1',
106 'repo_name': 'chromium' 109 'repo_name': 'chromium'
107 } 110 }
108 } 111 }
109 } 112 }
110 113
111 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus( 114 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus(
112 analysis, result) 115 analysis, result)
113 116
114 self.assertEqual(status, result_status.FOUND_UNTRIAGED) 117 self.assertEqual(status, result_status.FOUND_UNTRIAGED)
115 118
116 def testGetResultAnalysisStatusWithTryJobCulpritNotFoundCorrect(self): 119 def testGetResultAnalysisStatusWithTryJobCulpritNotFoundCorrect(self):
117 # Heuristic analysis found no results, which was correct. In this case, the 120 # Heuristic analysis found no results, which was correct. In this case, the
118 # try job result is actually a false positive. 121 # try job result is actually a false positive.
119 analysis = WfAnalysis.Create('m', 'b', 1) 122 analysis = WfAnalysis.Create('m', 'b', 1)
120 analysis.result_status = result_status.NOT_FOUND_CORRECT 123 analysis.result_status = result_status.NOT_FOUND_CORRECT
121 analysis.put() 124 analysis.put()
122 125
123 result = { 126 result = {
124 'culprit': { 127 'culprit': {
125 'compile': { 128 'compile': {
126 'revision': 'rev1', 129 'revision': 'rev1',
127 'commit_position': '1', 130 'commit_position': 1,
128 'url': 'url_1', 131 'url': 'url_1',
129 'repo_name': 'chromium' 132 'repo_name': 'chromium'
130 } 133 }
131 } 134 }
132 } 135 }
133 136
134 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus( 137 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus(
135 analysis, result) 138 analysis, result)
136 139
137 self.assertEqual(status, result_status.FOUND_UNTRIAGED) 140 self.assertEqual(status, result_status.FOUND_UNTRIAGED)
138 141
139 def testGetResultanalysisStatusWithTryJobCulpritNotFoundIncorrect(self): 142 def testGetResultanalysisStatusWithTryJobCulpritNotFoundIncorrect(self):
140 # Heuristic analysis found no results and was triaged to incorrect before a 143 # Heuristic analysis found no results and was triaged to incorrect before a
141 # try job result was found. In this case the try job result should override 144 # try job result was found. In this case the try job result should override
142 # the heuristic result. 145 # the heuristic result.
143 analysis = WfAnalysis.Create('m', 'b', 1) 146 analysis = WfAnalysis.Create('m', 'b', 1)
144 analysis.result_status = result_status.NOT_FOUND_INCORRECT 147 analysis.result_status = result_status.NOT_FOUND_INCORRECT
145 analysis.put() 148 analysis.put()
146 149
147 result = { 150 result = {
148 'culprit': { 151 'culprit': {
149 'compile': { 152 'compile': {
150 'revision': 'rev1', 153 'revision': 'rev1',
151 'commit_position': '1', 154 'commit_position': 1,
152 'url': 'url_1', 155 'url': 'url_1',
153 'repo_name': 'chromium' 156 'repo_name': 'chromium'
154 } 157 }
155 } 158 }
156 } 159 }
157 160
158 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus( 161 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus(
159 analysis, result) 162 analysis, result)
160 163
161 self.assertEqual(status, result_status.FOUND_UNTRIAGED) 164 self.assertEqual(status, result_status.FOUND_UNTRIAGED)
162 165
163 def testGetResultanalysisStatusWithTryJobCulpritNoHeuristicResult(self): 166 def testGetResultanalysisStatusWithTryJobCulpritNoHeuristicResult(self):
164 # In this case, the try job found a result before the heuristic result is 167 # In this case, the try job found a result before the heuristic result is
165 # available. This case should generally never happen, as heuristic analysis 168 # available. This case should generally never happen, as heuristic analysis
166 # is usually much faster than try jobs. 169 # is usually much faster than try jobs.
167 analysis = WfAnalysis.Create('m', 'b', 1) 170 analysis = WfAnalysis.Create('m', 'b', 1)
168 analysis.put() 171 analysis.put()
169 172
170 result = { 173 result = {
171 'culprit': { 174 'culprit': {
172 'compile': { 175 'compile': {
173 'revision': 'rev1', 176 'revision': 'rev1',
174 'commit_position': '1', 177 'commit_position': 1,
175 'url': 'url_1', 178 'url': 'url_1',
176 'repo_name': 'chromium' 179 'repo_name': 'chromium'
177 } 180 }
178 } 181 }
179 } 182 }
180 183
181 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus( 184 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus(
182 analysis, result) 185 analysis, result)
183 186
184 self.assertEqual(status, result_status.FOUND_UNTRIAGED) 187 self.assertEqual(status, result_status.FOUND_UNTRIAGED)
(...skipping 14 matching lines...) Expand all
199 # In this case, heuristic analysis found the correct culprit. The try job 202 # In this case, heuristic analysis found the correct culprit. The try job
200 # result should not overwrite it. 203 # result should not overwrite it.
201 analysis = WfAnalysis.Create('m', 'b', 1) 204 analysis = WfAnalysis.Create('m', 'b', 1)
202 analysis.result_status = result_status.FOUND_CORRECT 205 analysis.result_status = result_status.FOUND_CORRECT
203 analysis.put() 206 analysis.put()
204 207
205 result = { 208 result = {
206 'culprit': { 209 'culprit': {
207 'compile': { 210 'compile': {
208 'revision': 'rev1', 211 'revision': 'rev1',
209 'commit_position': '1', 212 'commit_position': 1,
210 'url': 'url_1', 213 'url': 'url_1',
211 'repo_name': 'chromium' 214 'repo_name': 'chromium'
212 } 215 }
213 } 216 }
214 } 217 }
215 218
216 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus( 219 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus(
217 analysis, result) 220 analysis, result)
218 self.assertEqual(status, result_status.FOUND_CORRECT) 221 self.assertEqual(status, result_status.FOUND_CORRECT)
219 222
(...skipping 21 matching lines...) Expand all
241 244
242 result = {} 245 result = {}
243 246
244 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus( 247 status = identify_try_job_culprit_pipeline._GetResultAnalysisStatus(
245 analysis, result) 248 analysis, result)
246 self.assertEqual(status, result_status.NOT_FOUND_INCORRECT) 249 self.assertEqual(status, result_status.NOT_FOUND_INCORRECT)
247 250
248 def testGetSuspectedCLsForCompileTryJob(self): 251 def testGetSuspectedCLsForCompileTryJob(self):
249 heuristic_suspected_cl = { 252 heuristic_suspected_cl = {
250 'revision': 'rev1', 253 'revision': 'rev1',
251 'commit_position': '1', 254 'commit_position': 1,
252 'url': 'url_1', 255 'url': 'url_1',
253 'repo_name': 'chromium' 256 'repo_name': 'chromium',
257 'failures': {'compile': []},
258 'top_score': 5
254 } 259 }
255 260
256 compile_suspected_cl = { 261 compile_suspected_cl = {
257 'revision': 'rev2', 262 'revision': 'rev2',
258 'commit_position': '2', 263 'commit_position': 2,
259 'url': 'url_2', 264 'url': 'url_2',
260 'repo_name': 'chromium' 265 'repo_name': 'chromium'
261 } 266 }
262 267
268 try_job_type = failure_type.COMPILE
269
263 analysis = WfAnalysis.Create('m', 'b', 1) 270 analysis = WfAnalysis.Create('m', 'b', 1)
264 analysis.suspected_cls = [heuristic_suspected_cl] 271 analysis.suspected_cls = [heuristic_suspected_cl]
265 analysis.put() 272 analysis.put()
266 273
267 result = { 274 try_job_suspected_cls = {
268 'culprit': { 275 'rev2': compile_suspected_cl
269 'compile': compile_suspected_cl
270 }
271 } 276 }
272 277
278 expected_cls = [
279 heuristic_suspected_cl,
280 {
281 'revision': 'rev2',
282 'commit_position': 2,
283 'url': 'url_2',
284 'repo_name': 'chromium',
285 'failures': {'compile': []},
286 'top_score': None
287 }
288 ]
289
273 self.assertEqual( 290 self.assertEqual(
274 identify_try_job_culprit_pipeline._GetSuspectedCLs(analysis, result), 291 identify_try_job_culprit_pipeline._GetSuspectedCLs(
275 [heuristic_suspected_cl, compile_suspected_cl]) 292 analysis, try_job_type, None, try_job_suspected_cls),
293 expected_cls)
276 294
277 def testGetSuspectedCLsForTestTryJobAndHeuristicResultsSame(self): 295 def testGetSuspectedCLsForTestTryJobAndHeuristicResultsSame(self):
278 suspected_cl = { 296 suspected_cl = {
279 'revision': 'rev1', 297 'revision': 'rev1',
280 'commit_position': '1', 298 'commit_position': 1,
281 'url': 'url_1', 299 'url': 'url_1',
282 'repo_name': 'chromium' 300 'repo_name': 'chromium'
283 } 301 }
284 302
285 analysis = WfAnalysis.Create('m', 'b', 1) 303 analysis = WfAnalysis.Create('m', 'b', 1)
286 analysis.suspected_cls = [suspected_cl] 304 analysis.suspected_cls = [suspected_cl]
287 analysis.put() 305 analysis.put()
288 306
289 result = { 307 try_job_suspected_cls = {
290 'culprit': { 308 'rev1': suspected_cl
291 'compile': suspected_cl
292 }
293 } 309 }
294 310
295 self.assertEqual( 311 updated_cls = identify_try_job_culprit_pipeline._GetSuspectedCLs(
296 identify_try_job_culprit_pipeline._GetSuspectedCLs(analysis, result), 312 analysis, failure_type.TEST, None, try_job_suspected_cls)
297 [suspected_cl]) 313 self.assertEqual(updated_cls, [suspected_cl])
298 314
299 def testGetSuspectedCLsForTestTryJob(self): 315 def testGetSuspectedCLsForTestTryJob(self):
300 suspected_cl1 = { 316 suspected_cl1 = {
301 'revision': 'rev1', 317 'revision': 'rev1',
302 'commit_position': '1', 318 'commit_position': 1,
303 'url': 'url_1', 319 'url': 'url_1',
304 'repo_name': 'chromium' 320 'repo_name': 'chromium'
305 } 321 }
306 suspected_cl2 = { 322 suspected_cl2 = {
307 'revision': 'rev2', 323 'revision': 'rev2',
308 'commit_position': '2', 324 'commit_position': 2,
309 'url': 'url_2', 325 'url': 'url_2',
310 'repo_name': 'chromium' 326 'repo_name': 'chromium'
311 } 327 }
312 suspected_cl3 = { 328 suspected_cl3 = {
313 'revision': 'rev3', 329 'revision': 'rev3',
314 'commit_position': '3', 330 'commit_position': 3,
315 'url': 'url_3', 331 'url': 'url_3',
316 'repo_name': 'chromium' 332 'repo_name': 'chromium'
317 } 333 }
318 334
319 analysis = WfAnalysis.Create('m', 'b', 1) 335 analysis = WfAnalysis.Create('m', 'b', 1)
320 analysis.suspected_cls = [] 336 analysis.suspected_cls = [suspected_cl3]
321 analysis.put() 337 analysis.put()
322 338
339 try_job_suspected_cls = {
340 'rev1': suspected_cl1,
341 'rev2': suspected_cl2
342 }
343
323 result = { 344 result = {
324 'culprit': { 345 'report': {
325 'a_test': { 346 'result': {
326 'tests': { 347 'rev1': {
327 'a_test1': suspected_cl1, 348 'step1': {
328 'a_test2': suspected_cl1 349 'status': 'failed',
350 'valid': True,
351 'failures': ['test1']
352 }
353 },
354 'rev2': {
355 'step1': {
356 'status': 'failed',
357 'valid': True,
358 'failures': ['test2']
359 }
329 } 360 }
330 },
331 'b_test': {
332 'tests': {
333 'b_test1': suspected_cl2
334 }
335 },
336 'c_test': {
337 'revision': 'rev3',
338 'commit_position': '3',
339 'url': 'url_3',
340 'repo_name': 'chromium',
341 'tests': {}
342 } 361 }
343 } 362 }
344 } 363 }
345 364
346 self.assertEqual( 365 expected_cls = [
347 identify_try_job_culprit_pipeline._GetSuspectedCLs(analysis, result), 366 suspected_cl3,
348 [suspected_cl3, suspected_cl2, suspected_cl1]) 367 {
368 'revision': 'rev1',
369 'commit_position': 1,
370 'url': 'url_1',
371 'repo_name': 'chromium',
372 'failures': {
373 'step1': ['test1']
374 },
375 'top_score': None
376 },
377 {
378 'revision': 'rev2',
379 'commit_position': 2,
380 'url': 'url_2',
381 'repo_name': 'chromium',
382 'failures': {
383 'step1': ['test2']
384 },
385 'top_score': None
386 }
387 ]
388
389 cl_result = identify_try_job_culprit_pipeline._GetSuspectedCLs(
390 analysis, failure_type.TEST, result, try_job_suspected_cls)
391 self.assertEqual(cl_result, expected_cls)
349 392
350 def testGetSuspectedCLsForTestTryJobWithHeuristicResult(self): 393 def testGetSuspectedCLsForTestTryJobWithHeuristicResult(self):
351 suspected_cl = { 394 suspected_cl = {
352 'revision': 'rev1', 395 'revision': 'rev1',
353 'commit_position': '1', 396 'commit_position': 1,
354 'url': 'url_1', 397 'url': 'url_1',
355 'repo_name': 'chromium' 398 'repo_name': 'chromium',
399 'failures': {
400 'step1': ['test1']
401 },
402 'top_score': 2
356 } 403 }
357 404
358 analysis = WfAnalysis.Create('m', 'b', 1) 405 analysis = WfAnalysis.Create('m', 'b', 1)
359 analysis.suspected_cls = [suspected_cl] 406 analysis.suspected_cls = [suspected_cl]
360 analysis.put() 407 analysis.put()
361 408
362 result = { 409 result = {
363 'culprit': { 410 'report': {
364 'a_test': { 411 'result': {
365 'revision': 'rev1', 412 'rev1': {
366 'commit_position': '1', 413 'step1': {
367 'url': 'url_1', 414 'status': 'failed',
368 'repo_name': 'chromium', 415 'valid': True,
369 'tests': {} 416 'failures': ['test1']
417 }
418 }
370 } 419 }
371 } 420 }
372 } 421 }
422
373 self.assertEqual( 423 self.assertEqual(
374 identify_try_job_culprit_pipeline._GetSuspectedCLs(analysis, result), 424 identify_try_job_culprit_pipeline._GetSuspectedCLs(
425 analysis, failure_type.TEST, result, {}),
375 [suspected_cl]) 426 [suspected_cl])
376 427
377 def testIdentifyCulpritForCompileTryJobNoCulprit(self): 428 def testIdentifyCulpritForCompileTryJobNoCulprit(self):
378 master_name = 'm' 429 master_name = 'm'
379 builder_name = 'b' 430 builder_name = 'b'
380 build_number = 1 431 build_number = 1
381 try_job_id = '1' 432 try_job_id = '1'
382 433
383 try_job = WfTryJob.Create(master_name, builder_name, build_number) 434 try_job = WfTryJob.Create(master_name, builder_name, build_number)
384 try_job.put() 435 try_job.put()
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
435 analysis.put() 486 analysis.put()
436 487
437 pipeline = IdentifyTryJobCulpritPipeline() 488 pipeline = IdentifyTryJobCulpritPipeline()
438 culprit = pipeline.run( 489 culprit = pipeline.run(
439 master_name, builder_name, build_number, ['rev1'], 490 master_name, builder_name, build_number, ['rev1'],
440 failure_type.COMPILE, '1', compile_result) 491 failure_type.COMPILE, '1', compile_result)
441 492
442 expected_culprit = 'rev2' 493 expected_culprit = 'rev2'
443 expected_suspected_cl = { 494 expected_suspected_cl = {
444 'revision': 'rev2', 495 'revision': 'rev2',
445 'commit_position': '2', 496 'commit_position': 2,
446 'url': 'url_2', 497 'url': 'url_2',
447 'repo_name': 'chromium' 498 'repo_name': 'chromium'
448 } 499 }
449 expected_compile_result = { 500 expected_compile_result = {
450 'report': { 501 'report': {
451 'result': { 502 'result': {
452 'rev1': 'passed', 503 'rev1': 'passed',
453 'rev2': 'failed' 504 'rev2': 'failed'
454 } 505 }
455 }, 506 },
456 'try_job_id': try_job_id, 507 'try_job_id': try_job_id,
457 'culprit': { 508 'culprit': {
458 'compile': expected_suspected_cl 509 'compile': expected_suspected_cl
459 } 510 }
460 } 511 }
512 expected_analysis_suspected_cls = [{
513 'revision': 'rev2',
514 'commit_position': 2,
515 'url': 'url_2',
516 'repo_name': 'chromium',
517 'failures': {'compile': []},
518 'top_score': None
519 }]
461 520
521 import json
522 print json.dumps(culprit, indent=2)
462 self.assertEqual(expected_compile_result['culprit'], culprit) 523 self.assertEqual(expected_compile_result['culprit'], culprit)
463 524
464 try_job = WfTryJob.Get(master_name, builder_name, build_number) 525 try_job = WfTryJob.Get(master_name, builder_name, build_number)
465 self.assertEqual(expected_compile_result, try_job.compile_results[-1]) 526 self.assertEqual(expected_compile_result, try_job.compile_results[-1])
466 self.assertEqual(analysis_status.COMPLETED, try_job.status) 527 self.assertEqual(analysis_status.COMPLETED, try_job.status)
467 528
468 try_job_data = WfTryJobData.Get(try_job_id) 529 try_job_data = WfTryJobData.Get(try_job_id)
469 analysis = WfAnalysis.Get(master_name, builder_name, build_number) 530 analysis = WfAnalysis.Get(master_name, builder_name, build_number)
470 self.assertEqual({'compile': expected_culprit}, try_job_data.culprits) 531 self.assertEqual({'compile': expected_culprit}, try_job_data.culprits)
471 self.assertEqual(analysis.result_status, 532 self.assertEqual(analysis.result_status,
472 result_status.FOUND_UNTRIAGED) 533 result_status.FOUND_UNTRIAGED)
473 self.assertEqual(analysis.suspected_cls, 534 self.assertEqual(analysis.suspected_cls, expected_analysis_suspected_cls)
474 [expected_suspected_cl])
475 535
476 def testIdentifyCulpritForCompileReturnNoneIfAllPassed(self): 536 def testIdentifyCulpritForCompileReturnNoneIfAllPassed(self):
477 master_name = 'm' 537 master_name = 'm'
478 builder_name = 'b' 538 builder_name = 'b'
479 build_number = 1 539 build_number = 1
480 try_job_id = '1' 540 try_job_id = '1'
481 541
482 compile_result = { 542 compile_result = {
483 'report': { 543 'report': {
484 'result': { 544 'result': {
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after
536 self.assertIsNone(analysis.suspected_cls) 596 self.assertIsNone(analysis.suspected_cls)
537 597
538 def testIdentifyCulpritForTestTryJobNoTryJobResultWithHeuristicResult(self): 598 def testIdentifyCulpritForTestTryJobNoTryJobResultWithHeuristicResult(self):
539 master_name = 'm' 599 master_name = 'm'
540 builder_name = 'b' 600 builder_name = 'b'
541 build_number = 1 601 build_number = 1
542 try_job_id = '1' 602 try_job_id = '1'
543 603
544 suspected_cl = { 604 suspected_cl = {
545 'revision': 'rev1', 605 'revision': 'rev1',
546 'commit_position': '1', 606 'commit_position': 1,
547 'url': 'url_1', 607 'url': 'url_1',
548 'repo_name': 'chromium' 608 'repo_name': 'chromium'
549 } 609 }
550 610
551 WfTryJobData.Create(try_job_id).put() 611 WfTryJobData.Create(try_job_id).put()
552 try_job = WfTryJob.Create(master_name, builder_name, build_number) 612 try_job = WfTryJob.Create(master_name, builder_name, build_number)
553 try_job.status = analysis_status.RUNNING 613 try_job.status = analysis_status.RUNNING
554 try_job.put() 614 try_job.put()
555 615
556 # Heuristic analysis already provided some results. 616 # Heuristic analysis already provided some results.
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
647 pipeline = IdentifyTryJobCulpritPipeline() 707 pipeline = IdentifyTryJobCulpritPipeline()
648 culprit = pipeline.run( 708 culprit = pipeline.run(
649 master_name, builder_name, build_number, ['rev3'], failure_type.TEST, 709 master_name, builder_name, build_number, ['rev3'], failure_type.TEST,
650 '1', test_result) 710 '1', test_result)
651 711
652 expected_suspected_cl = { 712 expected_suspected_cl = {
653 'revision': 'rev3', 713 'revision': 'rev3',
654 'repo_name': 'chromium' 714 'repo_name': 'chromium'
655 } 715 }
656 716
717 expected_analysis_suspected_cls = [
718 {
719 'revision': 'rev3',
720 'repo_name': 'chromium',
721 'failures': {'a_test' : ['a_test1']},
722 'top_score': None
723 }
724 ]
725
657 expected_culprit = { 726 expected_culprit = {
658 'a_test': { 727 'a_test': {
659 'tests': { 728 'tests': {
660 'a_test1': expected_suspected_cl 729 'a_test1': expected_suspected_cl
661 } 730 }
662 } 731 }
663 } 732 }
733
664 self.assertEqual(expected_culprit, culprit) 734 self.assertEqual(expected_culprit, culprit)
665 735
666 try_job_data = WfTryJobData.Get(try_job_id) 736 try_job_data = WfTryJobData.Get(try_job_id)
667 analysis = WfAnalysis.Get(master_name, builder_name, build_number) 737 analysis = WfAnalysis.Get(master_name, builder_name, build_number)
668 expected_culprit_data = { 738 expected_culprit_data = {
669 'a_test': { 739 'a_test': {
670 'a_test1': 'rev3' 740 'a_test1': 'rev3'
671 } 741 }
672 } 742 }
673 self.assertEqual(expected_culprit_data, try_job_data.culprits) 743 self.assertEqual(expected_culprit_data, try_job_data.culprits)
674 self.assertEqual(analysis.result_status, 744 self.assertEqual(analysis.result_status,
675 result_status.FOUND_UNTRIAGED) 745 result_status.FOUND_UNTRIAGED)
676 self.assertEqual(analysis.suspected_cls, [expected_suspected_cl]) 746 self.assertEqual(analysis.suspected_cls, expected_analysis_suspected_cls)
677 747
678 def testIdentifyCulpritForTestTryJobSuccess(self): 748 def testIdentifyCulpritForTestTryJobSuccess(self):
679 master_name = 'm' 749 master_name = 'm'
680 builder_name = 'b' 750 builder_name = 'b'
681 build_number = 1 751 build_number = 1
682 try_job_id = '1' 752 try_job_id = '1'
683 753
684 test_result = { 754 test_result = {
685 'report': { 755 'report': {
686 'result': { 756 'result': {
687 'rev1': { 757 'rev1': {
688 'a_test': { 758 'a_test': {
689 'status': 'failed', 759 'status': 'failed',
690 'valid': True, 760 'valid': True,
691 'failures': ['a_test1'] 761 'failures': ['a_test1']
692 }, 762 },
693 'b_test': { 763 'b_test': {
694 'status': 'failed', 764 'status': 'failed',
695 'valid': True, 765 'valid': True,
696 'failures': ['b_test1'] 766 'failures': ['b_test1']
697 },
698 'c_test': {
699 'status': 'passed',
700 'valid': True
701 } 767 }
702 }, 768 },
703 'rev2': { 769 'rev2': {
704 'a_test': { 770 'a_test': {
705 'status': 'failed', 771 'status': 'failed',
706 'valid': True, 772 'valid': True,
707 'failures': ['a_test1', 'a_test2'] 773 'failures': ['a_test1', 'a_test2']
708 }, 774 },
709 'b_test': { 775 'b_test': {
710 'status': 'passed', 776 'status': 'passed',
711 'valid': True 777 'valid': True
712 },
713 'c_test': {
714 'status': 'failed',
715 'valid': True,
716 'failures': []
717 } 778 }
718 } 779 }
719 } 780 }
720 }, 781 },
721 'url': 'url', 782 'url': 'url',
722 'try_job_id': try_job_id 783 'try_job_id': try_job_id
723 } 784 }
724 785
725 WfTryJobData.Create(try_job_id).put() 786 WfTryJobData.Create(try_job_id).put()
726 try_job = WfTryJob.Create(master_name, builder_name, build_number) 787 try_job = WfTryJob.Create(master_name, builder_name, build_number)
727 try_job.status = analysis_status.RUNNING 788 try_job.status = analysis_status.RUNNING
728 try_job.test_results = [test_result] 789 try_job.test_results = [test_result]
729 try_job.put() 790 try_job.put()
730 analysis = WfAnalysis.Create(master_name, builder_name, build_number) 791 analysis = WfAnalysis.Create(master_name, builder_name, build_number)
731 analysis.put() 792 analysis.put()
732 793
733 pipeline = IdentifyTryJobCulpritPipeline() 794 pipeline = IdentifyTryJobCulpritPipeline()
734 culprit = pipeline.run( 795 culprit = pipeline.run(
735 master_name, builder_name, build_number, ['rev1', 'rev2'], 796 master_name, builder_name, build_number, ['rev1', 'rev2'],
736 failure_type.TEST, '1', test_result) 797 failure_type.TEST, '1', test_result)
737 798
738 a_test1_suspected_cl = { 799 a_test1_suspected_cl = {
739 'revision': 'rev1', 800 'revision': 'rev1',
740 'commit_position': '1', 801 'commit_position': 1,
741 'url': 'url_1', 802 'url': 'url_1',
742 'repo_name': 'chromium' 803 'repo_name': 'chromium'
743 } 804 }
744 a_test2_suspected_cl = { 805 a_test2_suspected_cl = {
745 'revision': 'rev2', 806 'revision': 'rev2',
746 'commit_position': '2', 807 'commit_position': 2,
747 'url': 'url_2', 808 'url': 'url_2',
748 'repo_name': 'chromium' 809 'repo_name': 'chromium'
749 } 810 }
750 811
751 b_test1_suspected_cl = a_test1_suspected_cl 812 b_test1_suspected_cl = a_test1_suspected_cl
752 813
753 expected_test_result = { 814 expected_test_result = {
754 'report': { 815 'report': {
755 'result': { 816 'result': {
756 'rev1': { 817 'rev1': {
757 'a_test': { 818 'a_test': {
758 'status': 'failed', 819 'status': 'failed',
759 'valid': True, 820 'valid': True,
760 'failures': ['a_test1'] 821 'failures': ['a_test1']
761 }, 822 },
762 'b_test': { 823 'b_test': {
763 'status': 'failed', 824 'status': 'failed',
764 'valid': True, 825 'valid': True,
765 'failures': ['b_test1'] 826 'failures': ['b_test1']
766 },
767 'c_test': {
768 'status': 'passed',
769 'valid': True
770 } 827 }
771 }, 828 },
772 'rev2': { 829 'rev2': {
773 'a_test': { 830 'a_test': {
774 'status': 'failed', 831 'status': 'failed',
775 'valid': True, 832 'valid': True,
776 'failures': ['a_test1', 'a_test2'] 833 'failures': ['a_test1', 'a_test2']
777 }, 834 },
778 'b_test': { 835 'b_test': {
779 'status': 'passed', 836 'status': 'passed',
780 'valid': True 837 'valid': True
781 },
782 'c_test': {
783 'status': 'failed',
784 'valid': True,
785 'failures': []
786 } 838 }
787 } 839 }
788 } 840 }
789 }, 841 },
790 'url': 'url', 842 'url': 'url',
791 'try_job_id': try_job_id, 843 'try_job_id': try_job_id,
792 'culprit': { 844 'culprit': {
793 'a_test': { 845 'a_test': {
794 'tests': { 846 'tests': {
795 'a_test1': a_test1_suspected_cl, 847 'a_test1': a_test1_suspected_cl,
796 'a_test2': a_test2_suspected_cl 848 'a_test2': a_test2_suspected_cl
797 } 849 }
798 }, 850 },
799 'b_test': { 851 'b_test': {
800 'tests': { 852 'tests': {
801 'b_test1': b_test1_suspected_cl 853 'b_test1': b_test1_suspected_cl
802 } 854 }
803 },
804 'c_test': {
805 'revision': 'rev2',
806 'commit_position': '2',
807 'url': 'url_2',
808 'repo_name': 'chromium',
809 'tests': {}
810 } 855 }
811 } 856 }
812 } 857 }
813 858
814 self.assertEqual(expected_test_result['culprit'], culprit) 859 self.assertEqual(expected_test_result['culprit'], culprit)
815 860
816 try_job = WfTryJob.Get(master_name, builder_name, build_number) 861 try_job = WfTryJob.Get(master_name, builder_name, build_number)
817 self.assertEqual(expected_test_result, try_job.test_results[-1]) 862 self.assertEqual(expected_test_result, try_job.test_results[-1])
818 self.assertEqual(analysis_status.COMPLETED, try_job.status) 863 self.assertEqual(analysis_status.COMPLETED, try_job.status)
819 864
820 try_job_data = WfTryJobData.Get(try_job_id) 865 try_job_data = WfTryJobData.Get(try_job_id)
821 analysis = WfAnalysis.Get(master_name, builder_name, build_number) 866 analysis = WfAnalysis.Get(master_name, builder_name, build_number)
822 expected_culprit_data = { 867 expected_culprit_data = {
823 'a_test': { 868 'a_test': {
824 'a_test1': 'rev1', 869 'a_test1': 'rev1',
825 'a_test2': 'rev2', 870 'a_test2': 'rev2',
826 }, 871 },
827 'b_test': { 872 'b_test': {
828 'b_test1': 'rev1', 873 'b_test1': 'rev1',
874 }
875 }
876
877 expected_cls = [
878 {
879 'revision': 'rev1',
880 'commit_position': 1,
881 'url': 'url_1',
882 'repo_name': 'chromium',
883 'failures': {
884 'a_test': ['a_test1'],
885 'b_test': ['b_test1'],
886 },
887 'top_score': None
829 }, 888 },
830 'c_test': 'rev2' 889 {
890 'revision': 'rev2',
891 'commit_position': 2,
892 'url': 'url_2',
893 'repo_name': 'chromium',
894 'failures': {
895 'a_test': ['a_test1', 'a_test2']
896 },
897 'top_score': None
831 } 898 }
899 ]
832 self.assertEqual(expected_culprit_data, try_job_data.culprits) 900 self.assertEqual(expected_culprit_data, try_job_data.culprits)
833 self.assertEqual(analysis.result_status, 901 self.assertEqual(analysis.result_status,
834 result_status.FOUND_UNTRIAGED) 902 result_status.FOUND_UNTRIAGED)
835 self.assertEqual(analysis.suspected_cls, 903 self.assertEqual(analysis.suspected_cls, expected_cls)
836 [a_test2_suspected_cl, a_test1_suspected_cl])
837 904
838 def testAnalysisIsUpdatedOnlyIfStatusOrSuspectedCLsChanged(self): 905 def testAnalysisIsUpdatedOnlyIfStatusOrSuspectedCLsChanged(self):
839 master_name = 'm' 906 master_name = 'm'
840 builder_name = 'b' 907 builder_name = 'b'
841 build_number = 1 908 build_number = 1
842 try_job_id = '1' 909 try_job_id = '1'
910 repo_name = 'chromium'
911 revision = 'rev1'
912 commit_position = 1
843 913
844 suspected_cl = { 914 heuristic_suspected_cl = {
845 'revision': 'rev1', 915 'revision': revision,
846 'commit_position': '1', 916 'commit_position': commit_position,
847 'url': 'url_1', 917 'url': 'url_1',
848 'repo_name': 'chromium' 918 'repo_name': repo_name
849 } 919 }
850 920
851 analysis = WfAnalysis.Create(master_name, builder_name, build_number) 921 analysis = WfAnalysis.Create(master_name, builder_name, build_number)
852 analysis.suspected_cls = [suspected_cl] 922 analysis.suspected_cls = [heuristic_suspected_cl]
853 analysis.result_status = result_status.FOUND_UNTRIAGED 923 analysis.result_status = result_status.FOUND_UNTRIAGED
854 analysis.put() 924 analysis.put()
855 version = analysis.version 925 version = analysis.version
926
927 build_key = BaseBuildModel.CreateBuildId(
928 master_name, builder_name, build_number)
929 suspected_cl = WfSuspectedCL.Create(repo_name, revision, commit_position)
930 suspected_cl.approaches = [analysis_approach_type.HEURISTIC]
931 suspected_cl.builds = {
932 build_key: {
933 'approaches': [analysis_approach_type.HEURISTIC],
934 'failure_type':failure_type.COMPILE,
935 'failures': {'compile':[]},
936 'status': None,
937 'top_score': 4
938 }
939 }
940 suspected_cl.put()
941
856 compile_result = { 942 compile_result = {
857 'report': { 943 'report': {
858 'result': { 944 'result': {
859 'rev1': 'failed', 945 revision: 'failed',
860 }, 946 },
861 }, 947 },
862 } 948 }
863 949
864 try_job_data = WfTryJobData.Create(try_job_id) 950 try_job_data = WfTryJobData.Create(try_job_id)
865 try_job_data.put() 951 try_job_data.put()
866 952
867 try_job = WfTryJob.Create(master_name, builder_name, build_number) 953 try_job = WfTryJob.Create(master_name, builder_name, build_number)
868 try_job.status = analysis_status.RUNNING 954 try_job.status = analysis_status.RUNNING
869 try_job.compile_results = [{ 955 try_job.compile_results = [{
870 'report': { 956 'report': {
871 'result': { 957 'result': {
872 'rev1': 'failed', 958 revision: 'failed',
873 }, 959 },
874 }, 960 },
875 'try_job_id': try_job_id, 961 'try_job_id': try_job_id,
876 }] 962 }]
877 963
878 try_job.put() 964 try_job.put()
879 965
880 pipeline = IdentifyTryJobCulpritPipeline() 966 pipeline = IdentifyTryJobCulpritPipeline()
881 pipeline.run(master_name, builder_name, build_number, ['rev1'], 967 pipeline.run(master_name, builder_name, build_number, [revision],
882 failure_type.COMPILE, '1', compile_result) 968 failure_type.COMPILE, '1', compile_result)
883 969
884 self.assertEqual(analysis.result_status, 970 self.assertEqual(analysis.result_status,
885 result_status.FOUND_UNTRIAGED) 971 result_status.FOUND_UNTRIAGED)
886 self.assertEqual(analysis.suspected_cls, [suspected_cl]) 972 self.assertEqual(analysis.suspected_cls, [heuristic_suspected_cl])
887 self.assertEqual(version, analysis.version) # No update to analysis. 973 self.assertEqual(version, analysis.version) # No update to analysis.
888 974
975 expected_approaches = [
976 analysis_approach_type.HEURISTIC, analysis_approach_type.TRY_JOB]
977 expected_builds = {
978 build_key: {
979 'approaches': expected_approaches,
980 'failure_type':failure_type.COMPILE,
981 'failures': {'compile':[]},
982 'status': None,
983 'top_score': 4
984 }
985 }
986 suspected_cl = WfSuspectedCL.Get(repo_name, revision)
987 self.assertEqual(expected_approaches, suspected_cl.approaches)
988 self.assertEqual(expected_builds, suspected_cl.builds)
989
889 def testFindCulpritForEachTestFailureRevisionNotRun(self): 990 def testFindCulpritForEachTestFailureRevisionNotRun(self):
890 blame_list = ['rev1'] 991 blame_list = ['rev1']
891 result = { 992 result = {
892 'report': { 993 'report': {
893 'result': { 994 'result': {
894 'rev2': 'passed' 995 'rev2': 'passed'
895 } 996 }
896 } 997 }
897 } 998 }
898 999
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
939 try_job = WfTryJob.Create(master_name, builder_name, build_number).put() 1040 try_job = WfTryJob.Create(master_name, builder_name, build_number).put()
940 pipeline = IdentifyTryJobCulpritPipeline() 1041 pipeline = IdentifyTryJobCulpritPipeline()
941 culprit = pipeline.run(master_name, builder_name, build_number, ['rev1'], 1042 culprit = pipeline.run(master_name, builder_name, build_number, ['rev1'],
942 failure_type.TEST, None, None) 1043 failure_type.TEST, None, None)
943 self.assertIsNone(culprit) 1044 self.assertIsNone(culprit)
944 1045
945 try_job = WfTryJob.Get(master_name, builder_name, build_number) 1046 try_job = WfTryJob.Get(master_name, builder_name, build_number)
946 self.assertEqual(try_job.test_results, []) 1047 self.assertEqual(try_job.test_results, [])
947 self.assertEqual(try_job.status, analysis_status.COMPLETED) 1048 self.assertEqual(try_job.status, analysis_status.COMPLETED)
948 1049
949
950 def testNotifyCulprits(self): 1050 def testNotifyCulprits(self):
951 instances = [] 1051 instances = []
952 class Mocked_SendNotificationForCulpritPipeline(object): 1052 class Mocked_SendNotificationForCulpritPipeline(object):
953 def __init__(self, *args): 1053 def __init__(self, *args):
954 self.args = args 1054 self.args = args
955 self.started = False 1055 self.started = False
956 instances.append(self) 1056 instances.append(self)
957 1057
958 def start(self): 1058 def start(self):
959 self.started = True 1059 self.started = True
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after
1062 self.started = True 1162 self.started = True
1063 1163
1064 self.mock( 1164 self.mock(
1065 identify_try_job_culprit_pipeline, 'SendNotificationForCulpritPipeline', 1165 identify_try_job_culprit_pipeline, 'SendNotificationForCulpritPipeline',
1066 Mocked_SendNotificationForCulpritPipeline) 1166 Mocked_SendNotificationForCulpritPipeline)
1067 1167
1068 identify_try_job_culprit_pipeline._NotifyCulprits( 1168 identify_try_job_culprit_pipeline._NotifyCulprits(
1069 master_name, builder_name, build_number, None, 1169 master_name, builder_name, build_number, None,
1070 heuristic_cls, compile_suspected_cl) 1170 heuristic_cls, compile_suspected_cl)
1071 self.assertEqual(1, len(instances)) 1171 self.assertEqual(1, len(instances))
1072 self.assertTrue(instances[0].started) 1172 self.assertTrue(instances[0].started)
1173
1174 def testGetTestFailureCausedByCLResultNone(self):
1175 self.assertIsNone(
1176 identify_try_job_culprit_pipeline._GetTestFailureCausedByCL(None))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698