webkit/tools/layout_tests/update_expectations_from_dashboard_unittest.py - Issue 545145: Move the layout test scripts into a 'webkitpy' subdirectory in preparation...

Side by Side Diff: webkit/tools/layout_tests/update_expectations_from_dashboard_unittest.py

Issue 545145: Move the layout test scripts into a 'webkitpy' subdirectory in preparation... (Closed) Base URL: svn://chrome-svn/chrome/trunk/src/

Patch Set: try to de-confuse svn and the try bots Created 10 years, 11 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View unified diff | Download patch | Annotate | Revision Log

OLD	NEW
	(Empty)
1 #!/usr/bin/env python

2 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.

3 # Use of this source code is governed by a BSD-style license that can be

4 # found in the LICENSE file.

5

6 """Unittests to make sure we generate and update the expected-*.txt files

7 properly after running layout tests."""

8

9 import os

10 import sys

11 import unittest

12

13 import update_expectations_from_dashboard

14

15

16 class UpdateExpectationsUnittest(unittest.TestCase):

17 ###########################################################################

18 # Tests

19

20 def testKeepsUnmodifiedLines(self):

21 expectations = """// Ensure comments and newlines don't get stripped.

22 BUG1 SLOW : 1.html = PASS

23

24 BUG2 : 2.html = FAIL TIMEOUT

25 """

26 exp_results = """// Ensure comments and newlines don't get stripped.

27 BUG1 SLOW : 1.html = PASS

28

29 BUG2 : 2.html = FAIL TIMEOUT

30 """

31

32 updates = []

33 self.updateExpectations(expectations, updates, exp_results)

34

35 def testRemoveFlakyExpectation(self):

36 expectations = "BUG1 : 1.html = TIMEOUT FAIL\n"

37 expected_results = "BUG1 : 1.html = TIMEOUT\n"

38 updates = {"1.html": {

39 "WIN RELEASE": {"extra": "FAIL"},

40 "WIN DEBUG": {"extra": "FAIL"},

41 "LINUX RELEASE": {"extra": "FAIL"},

42 "LINUX DEBUG": {"extra": "FAIL"},

43 "MAC RELEASE": {"extra": "FAIL"},

44 "MAC DEBUG": {"extra": "FAIL"}}}

45 self.updateExpectations(expectations, updates, expected_results)

46

47 def testRemoveExpectationSlowTest(self):

48 expectations = "BUG1 SLOW : 1.html = FAIL\n"

49 expected_results = "BUG1 SLOW : 1.html = PASS\n"

50 updates = {"1.html": {

51 "WIN RELEASE": {"extra": "FAIL"},

52 "WIN DEBUG": {"extra": "FAIL"},

53 "LINUX RELEASE": {"extra": "FAIL"},

54 "LINUX DEBUG": {"extra": "FAIL"},

55 "MAC RELEASE": {"extra": "FAIL"},

56 "MAC DEBUG": {"extra": "FAIL"}}}

57 self.updateExpectations(expectations, updates, expected_results)

58

59 def testRemoveExpectation(self):

60 expectations = "BUG1 : 1.html = FAIL\n"

61 expected_results = ""

62 updates = {"1.html": {

63 "WIN RELEASE": {"extra": "FAIL"},

64 "WIN DEBUG": {"extra": "FAIL"},

65 "LINUX RELEASE": {"extra": "FAIL"},

66 "LINUX DEBUG": {"extra": "FAIL"},

67 "MAC RELEASE": {"extra": "FAIL"},

68 "MAC DEBUG": {"extra": "FAIL"}}}

69 self.updateExpectations(expectations, updates, expected_results)

70

71 def testRemoveExpectationFromOnePlatform(self):

72 expectations = "BUG1 : 1.html = FAIL\n"

73 expected_results = """BUG1 MAC WIN DEBUG : 1.html = FAIL

74 BUG1 RELEASE : 1.html = FAIL

75 """

76 updates = {"1.html": {"LINUX DEBUG": {"extra": "FAIL"}}}

77 self.updateExpectations(expectations, updates, expected_results)

78

79 def testRemoveSlow(self):

80 expectations = "BUG1 SLOW : 1.html = PASS\n"

81 expected_results = ""

82 updates = {"1.html": {

83 "WIN RELEASE": {"extra": "SLOW"},

84 "WIN DEBUG": {"extra": "SLOW"},

85 "LINUX RELEASE": {"extra": "SLOW"},

86 "LINUX DEBUG": {"extra": "SLOW"},

87 "MAC RELEASE": {"extra": "SLOW"},

88 "MAC DEBUG": {"extra": "SLOW"}}}

89 self.updateExpectations(expectations, updates, expected_results)

90

91 def testAddFlakyExpectation(self):

92 expectations = "BUG1 : 1.html = TIMEOUT\n"

93 expected_results = "BUG1 : 1.html = TIMEOUT FAIL\n"

94 updates = {"1.html": {

95 "WIN RELEASE": {"missing": "FAIL"},

96 "WIN DEBUG": {"missing": "FAIL"},

97 "LINUX RELEASE": {"missing": "FAIL"},

98 "LINUX DEBUG": {"missing": "FAIL"},

99 "MAC RELEASE": {"missing": "FAIL"},

100 "MAC DEBUG": {"missing": "FAIL"}}}

101 self.updateExpectations(expectations, updates, expected_results)

102

103 def testAddExpectationSlowTest(self):

104 expectations = "BUG1 SLOW : 1.html = PASS\n"

105 expected_results = "BUG1 SLOW : 1.html = PASS FAIL\n"

106 updates = {"1.html": {

107 "WIN RELEASE": {"missing": "FAIL"},

108 "WIN DEBUG": {"missing": "FAIL"},

109 "LINUX RELEASE": {"missing": "FAIL"},

110 "LINUX DEBUG": {"missing": "FAIL"},

111 "MAC RELEASE": {"missing": "FAIL"},

112 "MAC DEBUG": {"missing": "FAIL"}}}

113 self.updateExpectations(expectations, updates, expected_results)

114

115 def testAddExpectation(self):

116 # not yet implemented

117 return

118

119 expectations = ""

120 expected_results = "BUG1 : 1.html = FAIL\n"

121 updates = {"1.html": {

122 "WIN RELEASE": {"missing": "FAIL"},

123 "WIN DEBUG": {"missing": "FAIL"},

124 "LINUX RELEASE": {"missing": "FAIL"},

125 "LINUX DEBUG": {"missing": "FAIL"},

126 "MAC RELEASE": {"missing": "FAIL"},

127 "MAC DEBUG": {"missing": "FAIL"}}}

128 self.updateExpectations(expectations, updates, expected_results)

129

130 def testAddExpectationForOnePlatform(self):

131 expectations = "BUG1 WIN : 1.html = TIMEOUT\n"

132 expected_results = "BUG1 WIN : 1.html = TIMEOUT\n"

133 # TODO(ojan): Once we add currently unlisted tests, this expect results

134 # for this test should be:

135 #expected_results = """BUG1 WIN : 1.html = TIMEOUT

136 #BUG_AUTO LINUX DEBUG : 1.html = TIMEOUT

137 #"""

138 updates = {"1.html": {"LINUX DEBUG": {"missing": "TIMEOUT"}}}

139 self.updateExpectations(expectations, updates, expected_results)

140

141 def testAddSlow(self):

142 expectations = "BUG1 : 1.html = FAIL\n"

143 expected_results = "BUG1 SLOW : 1.html = FAIL\n"

144 updates = {"1.html": {

145 "WIN RELEASE": {"missing": "SLOW"},

146 "WIN DEBUG": {"missing": "SLOW"},

147 "LINUX RELEASE": {"missing": "SLOW"},

148 "LINUX DEBUG": {"missing": "SLOW"},

149 "MAC RELEASE": {"missing": "SLOW"},

150 "MAC DEBUG": {"missing": "SLOW"}}}

151 self.updateExpectations(expectations, updates, expected_results)

152

153 def testAddRemoveMultipleExpectations(self):

154 expectations = """BUG1 WIN : 1.html = FAIL

155 BUG2 MAC : 1.html = FAIL"""

156 expected_results = """BUG1 SLOW WIN : 1.html = FAIL

157 BUG2 MAC : 1.html = TIMEOUT\n"""

158 # TODO(ojan): Once we add currently unlisted tests, this expect results

159 # for this test should be:

160 #expected_results = """BUG1 SLOW WIN : 1.html = FAIL

161 #BUG_AUTO LINUX SLOW : 1.html = PASS

162 #BUG2 MAC : 1.html = TIMEOUT

163 #"""

164

165 updates = {"1.html": {

166 "WIN RELEASE": {"missing": "SLOW"},

167 "WIN DEBUG": {"missing": "SLOW"},

168 "LINUX RELEASE": {"missing": "SLOW"},

169 "LINUX DEBUG": {"missing": "SLOW"},

170 "MAC RELEASE": {"missing": "TIMEOUT", "extra": "FAIL"},

171 "MAC DEBUG": {"missing": "TIMEOUT", "extra": "FAIL"}}}

172 self.updateExpectations(expectations, updates, expected_results)

173

174 def testAddExistingExpectation(self):

175 expectations = "BUG1 : 1.html = FAIL\n"

176 expected_results = "BUG1 : 1.html = FAIL\n"

177 updates = {"1.html": {"WIN RELEASE": {"missing": "FAIL"}}}

178 self.updateExpectations(expectations, updates, expected_results)

179

180 def testAddImageOrTextToFailExpectation(self):

181 expectations = """BUG1 WIN RELEASE : 1.html = FAIL

182 BUG1 MAC RELEASE : 1.html = FAIL

183 BUG1 LINUX RELEASE : 1.html = FAIL

184 BUG1 LINUX DEBUG : 1.html = TIMEOUT

185 """

186 expected_results = """BUG1 WIN RELEASE : 1.html = IMAGE+TEXT

187 BUG1 MAC RELEASE : 1.html = IMAGE

188 BUG1 LINUX RELEASE : 1.html = TEXT

189 BUG1 LINUX DEBUG : 1.html = TIMEOUT IMAGE+TEXT

190 """

191 updates = {"1.html": {

192 "WIN RELEASE": {"missing": "IMAGE+TEXT"},

193 "MAC RELEASE": {"missing": "IMAGE"},

194 "LINUX RELEASE": {"missing": "TEXT"},

195 "LINUX DEBUG": {"missing": "IMAGE+TEXT"}}}

196 self.updateExpectations(expectations, updates, expected_results)

197

198 def testAddOther(self):

199 # Other is a catchall for more obscure expectations results.

200 # We should never add it to test_expectations.

201 expectations = "BUG1 WIN RELEASE : 1.html = FAIL\n"

202 expected_results = "BUG1 WIN RELEASE : 1.html = FAIL\n"

203 updates = {"1.html": {"WIN RELEASE": {"missing": "OTHER"}}}

204 self.updateExpectations(expectations, updates, expected_results)

205

206 def testRemoveNonExistantExpectation(self):

207 expectations = "BUG1 : 1.html = FAIL\n"

208 expected_results = "BUG1 : 1.html = FAIL\n"

209 updates = {"1.html": {"WIN RELEASE": {"extra": "TIMEOUT"}}}

210 self.updateExpectations(expectations, updates, expected_results)

211

212 def testUpdateSomePlatforms(self):

213 expectations = "BUG1 DEBUG : 1.html = TEXT PASS\n"

214 # TODO(ojan): Once we add currently unlisted tests, the expect results

215 # for this test should include the missing bits for RELEASE.

216 expected_results = "BUG1 LINUX DEBUG : 1.html = TEXT PASS\n"

217 updates = {"1.html": {

218 "WIN RELEASE": {"missing": "PASS TEXT"},

219 "WIN DEBUG": {"extra": "MISSING TEXT"},

220 "MAC RELEASE": {"missing": "PASS TEXT"},

221 "MAC DEBUG": {"extra": "MISSING TEXT"}}}

222 self.updateExpectations(expectations, updates, expected_results)

223

224 def testAddTimeoutToSlowTest(self):

225 # SLOW tests needing TIMEOUT need manual updating. Should just print

226 # a log and not modify the test.

227 expectations = "BUG1 SLOW : 1.html = TEXT\n"

228 expected_results = "BUG1 SLOW : 1.html = TEXT\n"

229 updates = {"1.html": {"WIN RELEASE": {"missing": "TIMEOUT"}}}

230 self.updateExpectations(expectations, updates, expected_results)

231

232 def testAddSlowToTimeoutTest(self):

233 # SLOW tests needing TIMEOUT need manual updating. Should just print

234 # a log and not modify the test.

235 expectations = "BUG1 : 1.html = TIMEOUT\n"

236 expected_results = "BUG1 : 1.html = TIMEOUT\n"

237 updates = {"1.html": {"WIN RELEASE": {"missing": "SLOW"}}}

238 self.updateExpectations(expectations, updates, expected_results)

239

240 def testIncludeLastPlatformInFlakiness(self):

241 # If a test is flaky on 5/6 platforms and the 6th's expectations are a

242 # subset of the other 5/6, then give them all the same expectations.

243 expectations = "BUG2 : 1.html = FAIL\n"

244 expected_results = "BUG2 : 1.html = FAIL TIMEOUT\n"

245 updates = {"1.html": {

246 "WIN RELEASE": {"missing": "TIMEOUT", "extra": "FAIL"},

247 "WIN DEBUG": {"missing": "TIMEOUT"},

248 "LINUX RELEASE": {"missing": "TIMEOUT"},

249 "LINUX DEBUG": {"missing": "TIMEOUT"},

250 "MAC RELEASE": {"missing": "TIMEOUT"},

251 "MAC DEBUG": {"missing": "TIMEOUT"}}}

252 self.updateExpectations(expectations, updates, expected_results)

253

254 def testIncludeLastPlatformInFlakinessThreeOutOfFour(self):

255 # If a test is flaky on 5/6 platforms and the 6th's expectations are a

256 # subset of the other 5/6, then give them all the same expectations.

257 expectations = "BUG2 MAC LINUX : 1.html = FAIL\n"

258 expected_results = "BUG2 LINUX MAC : 1.html = FAIL TIMEOUT\n"

259 updates = {"1.html": {

260 "LINUX RELEASE": {"missing": "TIMEOUT"},

261 "MAC RELEASE": {"missing": "TIMEOUT"},

262 "MAC DEBUG": {"missing": "TIMEOUT"}}}

263 self.updateExpectations(expectations, updates, expected_results)

264

265 def testExcludeLastPlatformFromFlakiness(self):

266 # If a test is flaky on 5/6 platforms and the 6th's expectations

267 # are not a subset of the other 5/6, then don't give them

268 # all the same expectations.

269 expectations = "BUG1 : 1.html = FAIL\n"

270 expected_results = """BUG1 DEBUG : 1.html = FAIL TIMEOUT

271 BUG1 LINUX MAC RELEASE : 1.html = FAIL TIMEOUT

272 BUG1 WIN RELEASE : 1.html = FAIL CRASH

273 """

274 updates = {"1.html": {

275 "WIN RELEASE": {"missing": "CRASH"},

276 "WIN DEBUG": {"missing": "TIMEOUT"},

277 "LINUX RELEASE": {"missing": "TIMEOUT"},

278 "LINUX DEBUG": {"missing": "TIMEOUT"},

279 "MAC RELEASE": {"missing": "TIMEOUT"},

280 "MAC DEBUG": {"missing": "TIMEOUT"}}}

281 self.updateExpectations(expectations, updates, expected_results)

282

283 def testStripComments(self):

284 expectations = """BUG1 : 1.html = TIMEOUT

285

286 // Comment/whitespace should be removed when the test is.

287 BUG2 WIN RELEASE : 2.html = TEXT

288

289 // Comment/whitespace after test should remain.

290

291 BUG2 MAC : 2.html = TEXT

292

293 // Comment/whitespace at end of file should remain.

294 """

295 expected_results = """BUG1 : 1.html = TIMEOUT

296

297 // Comment/whitespace after test should remain.

298

299 BUG2 MAC DEBUG : 2.html = TEXT

300

301 // Comment/whitespace at end of file should remain.

302 """

303 updates = {"2.html": {

304 "WIN RELEASE": {"extra": "TEXT"},

305 "MAC RELEASE": {"extra": "TEXT"}}}

306 self.updateExpectations(expectations, updates, expected_results)

307

308 def testLeaveComments(self):

309 expectations = """BUG1 : 1.html = TIMEOUT

310

311 // Comment/whitespace should remain.

312 BUG2 : 2.html = FAIL PASS

313 """

314 expected_results = """BUG1 : 1.html = TIMEOUT

315

316 // Comment/whitespace should remain.

317 BUG2 MAC DEBUG : 2.html = FAIL PASS

318 BUG2 LINUX MAC RELEASE : 2.html = FAIL PASS

319 """

320 updates = {"2.html": {

321 "WIN RELEASE": {"extra": "FAIL"},

322 "WIN DEBUG": {"extra": "FAIL"},

323 "LINUX DEBUG": {"extra": "FAIL"}}}

324 self.updateExpectations(expectations, updates, expected_results)

325

326 def testLeaveCommentsIfNoWhitespaceAfterTest(self):

327 expectations = """// Comment/whitespace should remain.

328 BUG2 WIN RELEASE : 2.html = TEXT

329 BUG2 : 1.html = IMAGE

330 """

331 expected_results = """// Comment/whitespace should remain.

332 BUG2 : 1.html = IMAGE

333 """

334 updates = {"2.html": {"WIN RELEASE": {"extra": "TEXT"}}}

335 self.updateExpectations(expectations, updates, expected_results)

336

337 def testLeavesUnmodifiedExpectationsUntouched(self):

338 # Ensures tests that would just change sort order of a line are noops.

339 expectations = "BUG1 WIN LINUX : 1.html = TIMEOUT\n"

340 expected_results = "BUG1 WIN LINUX : 1.html = TIMEOUT\n"

341 updates = {"1.html": {"MAC RELEASE": {"missing": "SLOW"}}}

342 self.updateExpectations(expectations, updates, expected_results)

343

344 ###########################################################################

345 # Helper functions

346

347 def updateExpectations(self, expectations, updates, expected_results):

348 results = update_expectations_from_dashboard.UpdateExpectations(

349 expectations, updates)

350 self.assertEqual(expected_results, results)

351

352 if '__main__' == __name__:

353 unittest.main()

OLD	NEW