sky/tools/webkitpy/tool/commands/rebaseline_unittest.py - Issue 946753002: Delete a bunch of dead python code.

Side by Side Diff: sky/tools/webkitpy/tool/commands/rebaseline_unittest.py

Issue 946753002: Delete a bunch of dead python code. (Closed) Base URL: git@github.com:domokit/mojo.git@master

Patch Set: Created 5 years, 10 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

OLD	NEW
	(Empty)
1 # Copyright (C) 2010 Google Inc. All rights reserved.

2 #

3 # Redistribution and use in source and binary forms, with or without

4 # modification, are permitted provided that the following conditions are

5 # met:

6 #

7 # * Redistributions of source code must retain the above copyright

8 # notice, this list of conditions and the following disclaimer.

9 # * Redistributions in binary form must reproduce the above

10 # copyright notice, this list of conditions and the following disclaimer

11 # in the documentation and/or other materials provided with the

12 # distribution.

13 # * Neither the name of Google Inc. nor the names of its

14 # contributors may be used to endorse or promote products derived from

15 # this software without specific prior written permission.

16 #

17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

28

29 import unittest

30

31 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer

32 from webkitpy.common.checkout.scm.scm_mock import MockSCM

33 from webkitpy.common.host_mock import MockHost

34 from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder

35 from webkitpy.common.net.layouttestresults import LayoutTestResults

36 from webkitpy.common.system.executive_mock import MockExecutive

37 from webkitpy.common.system.executive_mock import MockExecutive2

38 from webkitpy.common.system.outputcapture import OutputCapture

39 from webkitpy.tool.commands.rebaseline import *

40 from webkitpy.tool.mocktool import MockTool, MockOptions

41

42

43 class _BaseTestCase(unittest.TestCase):

44 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True'

45 WEB_PREFIX = 'http://example.com/f/builders/WebKit Mac10.7/results/layout-te st-results'

46

47 command_constructor = None

48

49 def setUp(self):

50 self.tool = MockTool()

51 self.command = self.command_constructor() # lint warns that command_con structor might not be set, but this is intentional; pylint: disable=E1102

52 self.command.bind_to_tool(self.tool)

53 self.lion_port = self.tool.port_factory.get_from_builder_name("WebKit Ma c10.7")

54 self.lion_expectations_path = self.lion_port.path_to_generic_test_expect ations_file()

55

56 # FIXME: crbug.com/279494. We should override builders._exact_matches

57 # here to point to a set of test ports and restore the value in

58 # tearDown(), and that way the individual tests wouldn't have to worry

59 # about it.

60

61 def _expand(self, path):

62 if self.tool.filesystem.isabs(path):

63 return path

64 return self.tool.filesystem.join(self.lion_port.layout_tests_dir(), path )

65

66 def _read(self, path):

67 return self.tool.filesystem.read_text_file(self._expand(path))

68

69 def _write(self, path, contents):

70 self.tool.filesystem.write_text_file(self._expand(path), contents)

71

72 def _zero_out_test_expectations(self):

73 for port_name in self.tool.port_factory.all_port_names():

74 port = self.tool.port_factory.get(port_name)

75 for path in port.expectations_files():

76 self._write(path, '')

77 self.tool.filesystem.written_files = {}

78

79 def _setup_mock_builder_data(self):

80 data = LayoutTestResults.results_from_string("""ADD_RESULTS({

81 "tests": {

82 "userscripts": {

83 "first-test.html": {

84 "expected": "PASS",

85 "actual": "IMAGE+TEXT"

86 },

87 "second-test.html": {

88 "expected": "FAIL",

89 "actual": "IMAGE+TEXT"

90 }

91 }

92 }

93 });""")

94 # FIXME: crbug.com/279494 - we shouldn't be mixing mock and real builder names.

95 for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7' ]:

96 self.command._builder_data[builder] = data

97

98

99 class TestCopyExistingBaselinesInternal(_BaseTestCase):

100 command_constructor = CopyExistingBaselinesInternal

101

102 def setUp(self):

103 super(TestCopyExistingBaselinesInternal, self).setUp()

104

105 def test_copying_overwritten_baseline(self):

106 self.tool.executive = MockExecutive2()

107

108 # FIXME: crbug.com/279494. it's confusing that this is the test- port, a nd not the regular lion port. Really all of the tests should be using the test p orts.

109 port = self.tool.port_factory.get('test-mac-snowleopard')

110 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/tes t-mac-snowleopard/failures/expected/image-expected.txt'), 'original snowleopard result')

111

112 old_exact_matches = builders._exact_matches

113 oc = OutputCapture()

114 try:

115 builders._exact_matches = {

116 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

117 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "speci fiers": set(["mock-specifier"])},

118 }

119

120 options = MockOptions(builder="MOCK SnowLeopard", suffixes="txt", ve rbose=True, test="failures/expected/image.html", results_directory=None)

121

122 oc.capture_output()

123 self.command.execute(options, [], self.tool)

124 finally:

125 out, _, _ = oc.restore_output()

126 builders._exact_matches = old_exact_matches

127

128 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layo ut_tests_dir(), 'platform/test-mac-leopard/failures/expected/image-expected.txt' )), 'original snowleopard result')

129 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete" : []}\n')

130

131 def test_copying_overwritten_baseline_to_multiple_locations(self):

132 self.tool.executive = MockExecutive2()

133

134 # FIXME: crbug.com/279494. it's confusing that this is the test- port, a nd not the regular win port. Really all of the tests should be using the test po rts.

135 port = self.tool.port_factory.get('test-win-win7')

136 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/tes t-win-win7/failures/expected/image-expected.txt'), 'original win7 result')

137

138 old_exact_matches = builders._exact_matches

139 oc = OutputCapture()

140 try:

141 builders._exact_matches = {

142 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

143 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": s et(["mock-specifier"])},

144 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set([" mock-specifier"])},

145 }

146

147 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=T rue, test="failures/expected/image.html", results_directory=None)

148

149 oc.capture_output()

150 self.command.execute(options, [], self.tool)

151 finally:

152 out, _, _ = oc.restore_output()

153 builders._exact_matches = old_exact_matches

154

155 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layo ut_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt ')), 'original win7 result')

156 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(p ort.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected. txt')))

157 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete" : []}\n')

158

159 def test_no_copy_existing_baseline(self):

160 self.tool.executive = MockExecutive2()

161

162 # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports.

163 port = self.tool.port_factory.get('test-win-win7')

164 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/tes t-win-win7/failures/expected/image-expected.txt'), 'original win7 result')

165

166 old_exact_matches = builders._exact_matches

167 oc = OutputCapture()

168 try:

169 builders._exact_matches = {

170 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

171 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": s et(["mock-specifier"])},

172 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set([" mock-specifier"])},

173 }

174

175 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=T rue, test="failures/expected/image.html", results_directory=None)

176

177 oc.capture_output()

178 self.command.execute(options, [], self.tool)

179 finally:

180 out, _, _ = oc.restore_output()

181 builders._exact_matches = old_exact_matches

182

183 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layo ut_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt ')), 'original win7 result')

184 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layo ut_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'original win7 result')

185 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(p ort.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected. txt')))

186 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete" : []}\n')

187

188 def test_no_copy_skipped_test(self):

189 self.tool.executive = MockExecutive2()

190

191 port = self.tool.port_factory.get('test-win-win7')

192 fs = self.tool.filesystem

193 self._write(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/fai lures/expected/image-expected.txt'), 'original win7 result')

194 expectations_path = fs.join(port.path_to_generic_test_expectations_file( ))

195 self._write(expectations_path, (

196 "[ Win ] failures/expected/image.html [ Failure ]\n"

197 "[ Linux ] failures/expected/image.html [ Skip ]\n"))

198 old_exact_matches = builders._exact_matches

199 oc = OutputCapture()

200 try:

201 builders._exact_matches = {

202 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": s et(["mock-specifier"])},

203 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

204 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set([" mock-specifier"])},

205 }

206

207 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=T rue, test="failures/expected/image.html", results_directory=None)

208

209 oc.capture_output()

210 self.command.execute(options, [], self.tool)

211 finally:

212 out, _, _ = oc.restore_output()

213 builders._exact_matches = old_exact_matches

214

215 self.assertFalse(fs.exists(fs.join(port.layout_tests_dir(), 'platform/te st-linux-x86_64/failures/expected/image-expected.txt')))

216 self.assertEqual(self._read(fs.join(port.layout_tests_dir(), 'platform/t est-win-win7/failures/expected/image-expected.txt')),

217 'original win7 result')

218

219

220 class TestRebaselineTest(_BaseTestCase):

221 command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-int ernal

222

223 def setUp(self):

224 super(TestRebaselineTest, self).setUp()

225 self.options = MockOptions(builder="WebKit Mac10.7", test="userscripts/a nother-test.html", suffixes="txt", results_directory=None)

226

227 def test_baseline_directory(self):

228 command = self.command

229 self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/third_party/WebKit/tests/platform/mac-lion")

230 self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/third_party/WebKit/tests/platform/mac-snowleopard")

231

232 def test_rebaseline_updates_expectations_file_noop(self):

233 self._zero_out_test_expectations()

234 self._write(self.lion_expectations_path, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]

235 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]

236 """)

237 self._write("fast/dom/Window/window-postmessage-clone-really-deep-array. html", "Dummy test contents")

238 self._write("fast/css/large-list-of-rules-crash.html", "Dummy test conte nts")

239 self._write("userscripts/another-test.html", "Dummy test contents")

240

241 self.options.suffixes = "png,wav,txt"

242 self.command._rebaseline_test_and_update_expectations(self.options)

243

244 self.assertItemsEqual(self.tool.web.urls_fetched,

245 [self.WEB_PREFIX + '/userscripts/another-test-actual.png',

246 self.WEB_PREFIX + '/userscripts/another-test-actual.wav',

247 self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])

248 new_expectations = self._read(self.lion_expectations_path)

249 self.assertMultiLineEqual(new_expectations, """Bug(B) [ Mac Linux XP Deb ug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]

250 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]

251 """)

252

253 def test_rebaseline_test(self):

254 self.command._rebaseline_test("WebKit Linux", "userscripts/another-test. html", "txt", self.WEB_PREFIX)

255 self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/u serscripts/another-test-actual.txt'])

256

257 def test_rebaseline_test_with_results_directory(self):

258 self._write("userscripts/another-test.html", "test data")

259 self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/ano ther-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.h tml [ ImageOnlyFailure ]\n")

260 self.options.results_directory = '/tmp'

261 self.command._rebaseline_test_and_update_expectations(self.options)

262 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscri pts/another-test-actual.txt'])

263

264 def test_rebaseline_reftest(self):

265 self._write("userscripts/another-test.html", "test data")

266 self._write("userscripts/another-test-expected.sky", "generic result")

267 OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_u pdate_expectations, args=[self.options],

268 expected_logs="Cannot rebaseline reftest: userscripts/another-test.h tml\n")

269 self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-line s': [], "delete": []})

270

271 def test_rebaseline_test_and_print_scm_changes(self):

272 self.command._print_scm_changes = True

273 self.command._scm_changes = {'add': [], 'delete': []}

274 self.tool._scm.exists = lambda x: False

275

276 self.command._rebaseline_test("WebKit Linux", "userscripts/another-test. html", "txt", None)

277

278 self.assertDictEqual(self.command._scm_changes, {'add': ['/mock-checkout /third_party/WebKit/tests/platform/linux/userscripts/another-test-expected.txt'] , 'delete': []})

279

280 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):

281 self.tool.executive = MockExecutive2()

282

283 # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports.

284 port = self.tool.port_factory.get('test-win-win7')

285 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/tes t-win-win7/failures/expected/image-expected.txt'), 'original win7 result')

286

287 old_exact_matches = builders._exact_matches

288 oc = OutputCapture()

289 try:

290 builders._exact_matches = {

291 "MOCK XP": {"port_name": "test-win-xp"},

292 "MOCK Win7": {"port_name": "test-win-win7"},

293 }

294

295 options = MockOptions(optimize=True, builder="MOCK Win7", suffixes=" txt",

296 verbose=True, test="failures/expected/image.html", results_direc tory=None)

297

298 oc.capture_output()

299 self.command.execute(options, [], self.tool)

300 finally:

301 out, _, _ = oc.restore_output()

302 builders._exact_matches = old_exact_matches

303

304 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layo ut_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True')

305 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(p ort.layout_tests_dir(), 'platform/test-win-xp/failures/expected/image-expected.t xt')))

306 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "f ailures/expected/image.html", "builder": "MOCK Win7"}], "delete": []}\n')

307

308

309 class TestAbstractParallelRebaselineCommand(_BaseTestCase):

310 command_constructor = AbstractParallelRebaselineCommand

311

312 def test_builders_to_fetch_from(self):

313 old_exact_matches = builders._exact_matches

314 try:

315 builders._exact_matches = {

316 "MOCK XP": {"port_name": "test-win-xp"},

317 "MOCK Win7": {"port_name": "test-win-win7"},

318 "MOCK Win7 (dbg)(1)": {"port_name": "test-win-win7"},

319 "MOCK Win7 (dbg)(2)": {"port_name": "test-win-win7"},

320 }

321

322 builders_to_fetch = self.command._builders_to_fetch_from(["MOCK XP", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win7"])

323 self.assertEqual(builders_to_fetch, ["MOCK XP", "MOCK Win7"])

324 finally:

325 builders._exact_matches = old_exact_matches

326

327

328 class TestRebaselineJson(_BaseTestCase):

329 command_constructor = RebaselineJson

330

331 def setUp(self):

332 super(TestRebaselineJson, self).setUp()

333 self.tool.executive = MockExecutive2()

334 self.old_exact_matches = builders._exact_matches

335 builders._exact_matches = {

336 "MOCK builder": {"port_name": "test-mac-snowleopard"},

337 "MOCK builder (Debug)": {"port_name": "test-mac-snowleopard"},

338 }

339

340 def tearDown(self):

341 builders._exact_matches = self.old_exact_matches

342 super(TestRebaselineJson, self).tearDown()

343

344 def test_rebaseline_test_passes_on_all_builders(self):

345 self._setup_mock_builder_data()

346

347 def builder_data():

348 self.command._builder_data['MOCK builder'] = LayoutTestResults.resul ts_from_string("""ADD_RESULTS({

349 "tests": {

350 "userscripts": {

351 "first-test.html": {

352 "expected": "NEEDSREBASELINE",

353 "actual": "PASS"

354 }

355 }

356 }

357 });""")

358 return self.command._builder_data

359

360 self.command.builder_data = builder_data

361

362 options = MockOptions(optimize=True, verbose=True, results_directory=Non e)

363

364 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test. html [ ImageOnlyFailure ]\n")

365 self._write("userscripts/first-test.html", "Dummy test contents")

366

367 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOC K builder": ["txt", "png"]}})

368

369 # Note that we have one run_in_parallel() call followed by a run_command ()

370 self.assertEqual(self.tool.executive.calls,

371 [[['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suf fixes', '', 'userscripts/first-test.html', '--verbose']]])

372

373 def test_rebaseline_all(self):

374 self._setup_mock_builder_data()

375

376 options = MockOptions(optimize=True, verbose=True, results_directory=Non e)

377 self._write("userscripts/first-test.html", "Dummy test contents")

378 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOC K builder": ["txt", "png"]}})

379

380 # Note that we have one run_in_parallel() call followed by a run_command ()

381 self.assertEqual(self.tool.executive.calls,

382 [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes ', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.htm l', '--verbose']],

383 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt, png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--v erbose']],

384 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suf fixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]])

385

386 def test_rebaseline_debug(self):

387 self._setup_mock_builder_data()

388

389 options = MockOptions(optimize=True, verbose=True, results_directory=Non e)

390 self._write("userscripts/first-test.html", "Dummy test contents")

391 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOC K builder (Debug)": ["txt", "png"]}})

392

393 # Note that we have one run_in_parallel() call followed by a run_command ()

394 self.assertEqual(self.tool.executive.calls,

395 [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes ', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first- test.html', '--verbose']],

396 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt, png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.htm l', '--verbose']],

397 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suf fixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]])

398

399 def test_no_optimize(self):

400 self._setup_mock_builder_data()

401

402 options = MockOptions(optimize=False, verbose=True, results_directory=No ne)

403 self._write("userscripts/first-test.html", "Dummy test contents")

404 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOC K builder (Debug)": ["txt", "png"]}})

405

406 # Note that we have only one run_in_parallel() call

407 self.assertEqual(self.tool.executive.calls,

408 [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes ', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first- test.html', '--verbose']],

409 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt, png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.htm l', '--verbose']]])

410

411 def test_results_directory(self):

412 self._setup_mock_builder_data()

413

414 options = MockOptions(optimize=False, verbose=True, results_directory='/ tmp')

415 self._write("userscripts/first-test.html", "Dummy test contents")

416 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOC K builder": ["txt", "png"]}})

417

418 # Note that we have only one run_in_parallel() call

419 self.assertEqual(self.tool.executive.calls,

420 [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes ', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.htm l', '--results-directory', '/tmp', '--verbose']],

421 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt, png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--r esults-directory', '/tmp', '--verbose']]])

422

423 class TestRebaselineJsonUpdatesExpectationsFiles(_BaseTestCase):

424 command_constructor = RebaselineJson

425

426 def setUp(self):

427 super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp()

428 self.tool.executive = MockExecutive2()

429

430 def mock_run_command(args,

431 cwd=None,

432 input=None,

433 error_handler=None,

434 return_exit_code=False,

435 return_stderr=True,

436 decode_output=False,

437 env=None):

438 return '{"add": [], "remove-lines": [{"test": "userscripts/first-tes t.html", "builder": "WebKit Mac10.7"}]}\n'

439 self.tool.executive.run_command = mock_run_command

440

441 def test_rebaseline_updates_expectations_file(self):

442 options = MockOptions(optimize=False, verbose=True, results_directory=No ne)

443

444 self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/fir st-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n")

445 self._write("userscripts/first-test.html", "Dummy test contents")

446 self._setup_mock_builder_data()

447

448 self.command._rebaseline(options, {"userscripts/first-test.html": {"Web Kit Mac10.7": ["txt", "png"]}})

449

450 new_expectations = self._read(self.lion_expectations_path)

451 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Mavericks Mountain Lion Retina SnowLeopard ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug( z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n")

452

453 def test_rebaseline_updates_expectations_file_all_platforms(self):

454 options = MockOptions(optimize=False, verbose=True, results_directory=No ne)

455

456 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test. html [ ImageOnlyFailure ]\n")

457 self._write("userscripts/first-test.html", "Dummy test contents")

458 self._setup_mock_builder_data()

459

460 self.command._rebaseline(options, {"userscripts/first-test.html": {"Web Kit Mac10.7": ["txt", "png"]}})

461

462 new_expectations = self._read(self.lion_expectations_path)

463 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Android Linux Mave ricks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageO nlyFailure ]\n")

464

465 def test_rebaseline_handles_platform_skips(self):

466 # This test is just like test_rebaseline_updates_expectations_file_all_p latforms(),

467 # except that if a particular port happens to SKIP a test in an override s file,

468 # we count that as passing, and do not think that we still need to rebas eline it.

469 options = MockOptions(optimize=False, verbose=True, results_directory=No ne)

470

471 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test. html [ ImageOnlyFailure ]\n")

472 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ Skip ]\n" )

473 self._write("userscripts/first-test.html", "Dummy test contents")

474 self._setup_mock_builder_data()

475

476 self.command._rebaseline(options, {"userscripts/first-test.html": {"Web Kit Mac10.7": ["txt", "png"]}})

477

478 new_expectations = self._read(self.lion_expectations_path)

479 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks Mo untainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailu re ]\n")

480

481 def test_rebaseline_handles_skips_in_file(self):

482 # This test is like test_Rebaseline_handles_platform_skips, except that the

483 # Skip is in the same (generic) file rather than a platform file. In thi s case,

484 # the Skip line should be left unmodified. Note that the first line is n ow

485 # qualified as "[Linux Mac Win]"; if it was unqualified, it would confli ct with

486 # the second line.

487 options = MockOptions(optimize=False, verbose=True, results_directory=No ne)

488

489 self._write(self.lion_expectations_path,

490 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ ImageOnlyFa ilure ]\n"

491 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"))

492 self._write("userscripts/first-test.html", "Dummy test contents")

493 self._setup_mock_builder_data()

494

495 self.command._rebaseline(options, {"userscripts/first-test.html": {"Web Kit Mac10.7": ["txt", "png"]}})

496

497 new_expectations = self._read(self.lion_expectations_path)

498 self.assertMultiLineEqual(new_expectations,

499 ("Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] use rscripts/first-test.html [ ImageOnlyFailure ]\n"

500 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"))

501

502 def test_rebaseline_handles_smoke_tests(self):

503 # This test is just like test_rebaseline_handles_platform_skips, except that we check for

504 # a test not being in the SmokeTests file, instead of using overrides fi les.

505 # If a test is not part of the smoke tests, we count that as passing on ports that only

506 # run smoke tests, and do not think that we still need to rebaseline it.

507 options = MockOptions(optimize=False, verbose=True, results_directory=No ne)

508

509 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test. html [ ImageOnlyFailure ]\n")

510 self._write("SmokeTests", "fast/html/article-element.html")

511 self._write("userscripts/first-test.html", "Dummy test contents")

512 self._setup_mock_builder_data()

513

514 self.command._rebaseline(options, {"userscripts/first-test.html": {"Web Kit Mac10.7": ["txt", "png"]}})

515

516 new_expectations = self._read(self.lion_expectations_path)

517 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks Mo untainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailu re ]\n")

518

519

520 class TestRebaseline(_BaseTestCase):

521 # This command shares most of its logic with RebaselineJson, so these tests just test what is different.

522

523 command_constructor = Rebaseline # AKA webkit-patch rebaseline

524

525 def test_rebaseline(self):

526 self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder ')]

527

528 self._write("userscripts/first-test.html", "test data")

529

530 self._zero_out_test_expectations()

531 self._setup_mock_builder_data()

532

533 old_exact_matches = builders._exact_matches

534 try:

535 builders._exact_matches = {

536 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

537 }

538 self.command.execute(MockOptions(results_directory=False, optimize=F alse, builders=None, suffixes="txt,png", verbose=True), ['userscripts/first-test .html'], self.tool)

539 finally:

540 builders._exact_matches = old_exact_matches

541

542 calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.t ool.executive.calls)

543 self.assertEqual(calls,

544 [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes ', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.htm l', '--verbose']],

545 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt, png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--v erbose']]])

546

547 def test_rebaseline_directory(self):

548 self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder ')]

549

550 self._write("userscripts/first-test.html", "test data")

551 self._write("userscripts/second-test.html", "test data")

552

553 self._setup_mock_builder_data()

554

555 old_exact_matches = builders._exact_matches

556 try:

557 builders._exact_matches = {

558 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

559 }

560 self.command.execute(MockOptions(results_directory=False, optimize=F alse, builders=None, suffixes="txt,png", verbose=True), ['userscripts'], self.to ol)

561 finally:

562 builders._exact_matches = old_exact_matches

563

564 calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.t ool.executive.calls)

565 self.assertEqual(calls,

566 [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes ', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.htm l', '--verbose'],

567 ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes ', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.ht ml', '--verbose']],

568 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt, png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--v erbose'],

569 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt, png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '-- verbose']]])

570

571

572 class MockLineRemovingExecutive(MockExecutive):

573 def run_in_parallel(self, commands):

574 assert len(commands)

575

576 num_previous_calls = len(self.calls)

577 command_outputs = []

578 for cmd_line, cwd in commands:

579 out = self.run_command(cmd_line, cwd=cwd)

580 if 'rebaseline-test-internal' in cmd_line:

581 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "% s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6])

582 command_outputs.append([0, out, ''])

583

584 new_calls = self.calls[num_previous_calls:]

585 self.calls = self.calls[:num_previous_calls]

586 self.calls.append(new_calls)

587 return command_outputs

588

589

590 class TestRebaselineExpectations(_BaseTestCase):

591 command_constructor = RebaselineExpectations

592

593 def setUp(self):

594 super(TestRebaselineExpectations, self).setUp()

595 self.options = MockOptions(optimize=False, builders=None, suffixes=['txt '], verbose=False, platform=None, results_directory=None)

596

597 def _write_test_file(self, port, path, contents):

598 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path)

599 self.tool.filesystem.write_text_file(abs_path, contents)

600

601 def _setup_test_port(self):

602 test_port = self.tool.port_factory.get('test')

603 original_get = self.tool.port_factory.get

604

605 def get_test_port(port_name=None, options=None, **kwargs):

606 if not port_name:

607 return test_port

608 return original_get(port_name, options, **kwargs)

609 # Need to make sure all the ports grabbed use the test checkout path ins tead of the mock checkout path.

610 # FIXME: crbug.com/279494 - we shouldn't be doing this.

611 self.tool.port_factory.get = get_test_port

612

613 return test_port

614

615 def test_rebaseline_expectations(self):

616 self._zero_out_test_expectations()

617

618 self.tool.executive = MockExecutive2()

619

620 def builder_data():

621 self.command._builder_data['MOCK SnowLeopard'] = self.command._build er_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({

622 "tests": {

623 "userscripts": {

624 "another-test.html": {

625 "expected": "PASS",

626 "actual": "PASS TEXT"

627 },

628 "images.svg": {

629 "expected": "FAIL",

630 "actual": "IMAGE+TEXT"

631 }

632 }

633 }

634 });""")

635 return self.command._builder_data

636

637 self.command.builder_data = builder_data

638

639 self._write("userscripts/another-test.html", "Dummy test contents")

640 self._write("userscripts/images.svg", "Dummy test contents")

641 self.command._tests_to_rebaseline = lambda port: {

642 'userscripts/another-test.html': set(['txt']),

643 'userscripts/images.svg': set(['png']),

644 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']),

645 }

646

647 old_exact_matches = builders._exact_matches

648 try:

649 builders._exact_matches = {

650 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

651 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "speci fiers": set(["mock-specifier"])},

652 }

653 self.command.execute(self.options, [], self.tool)

654 finally:

655 builders._exact_matches = old_exact_matches

656

657 # FIXME: change this to use the test- ports.

658 calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls )

659 self.assertEqual(self.tool.executive.calls, [

660 [

661 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix es', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.htm l'],

662 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix es', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test .html'],

663 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix es', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'],

664 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix es', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'] ,

665 ],

666 [

667 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'tx t', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'],

668 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'tx t', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'],

669 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'pn g', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'],

670 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'pn g', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'],

671 ],

672 ])

673

674 def test_rebaseline_expectations_noop(self):

675 self._zero_out_test_expectations()

676

677 oc = OutputCapture()

678 try:

679 oc.capture_output()

680 self.command.execute(self.options, [], self.tool)

681 finally:

682 _, _, logs = oc.restore_output()

683 self.assertEqual(self.tool.filesystem.written_files, {})

684 self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n' )

685

686 def disabled_test_overrides_are_included_correctly(self):

687 # This tests that the any tests marked as REBASELINE in the overrides ar e found, but

688 # that the overrides do not get written into the main file.

689 self._zero_out_test_expectations()

690

691 self._write(self.lion_expectations_path, '')

692 self.lion_port.expectations_dict = lambda: {

693 self.lion_expectations_path: '',

694 'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebase line ]\n'

695 'Bug(y) userscripts/test.html [ Crash ]\n')}

696 self._write('/userscripts/another-test.html', '')

697

698 self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': set(['png', 'txt', 'wav'])})

699 self.assertEqual(self._read(self.lion_expectations_path), '')

700

701 def test_rebaseline_without_other_expectations(self):

702 self._write("userscripts/another-test.html", "Dummy test contents")

703 self._write(self.lion_expectations_path, "Bug(x) userscripts/another-tes t.html [ Rebaseline ]\n")

704 self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': ('png', 'wav', 'txt')})

705

706 def test_rebaseline_test_passes_everywhere(self):

707 test_port = self._setup_test_port()

708

709 old_builder_data = self.command.builder_data

710

711 def builder_data():

712 self.command._builder_data['MOCK Leopard'] = self.command._builder_d ata['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({

713 "tests": {

714 "fast": {

715 "dom": {

716 "prototype-taco.html": {

717 "expected": "FAIL",

718 "actual": "PASS",

719 "is_unexpected": true

720 }

721 }

722 }

723 }

724 });""")

725 return self.command._builder_data

726

727 self.command.builder_data = builder_data

728

729 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expe ctations_file(), """

730 Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ]

731 """)

732

733 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")

734

735 self.tool.executive = MockLineRemovingExecutive()

736

737 old_exact_matches = builders._exact_matches

738 try:

739 builders._exact_matches = {

740 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

741 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "speci fiers": set(["mock-specifier"])},

742 }

743

744 self.command.execute(self.options, [], self.tool)

745 self.assertEqual(self.tool.executive.calls, [])

746

747 # The mac ports should both be removed since they're the only ones i n builders._exact_matches.

748 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_ to_generic_test_expectations_file()), """

749 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ]

750 """)

751 finally:

752 builders._exact_matches = old_exact_matches

753

754

755 class _FakeOptimizer(BaselineOptimizer):

756 def read_results_by_directory(self, baseline_name):

757 if baseline_name.endswith('txt'):

758 return {'tests/passes/text.html': '123456'}

759 return {}

760

761

762 class TestOptimizeBaselines(_BaseTestCase):

763 command_constructor = OptimizeBaselines

764

765 def _write_test_file(self, port, path, contents):

766 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path)

767 self.tool.filesystem.write_text_file(abs_path, contents)

768

769 def setUp(self):

770 super(TestOptimizeBaselines, self).setUp()

771

772 # FIXME: This is a hack to get the unittest and the BaselineOptimize to both use /mock-checkout

773 # instead of one using /mock-checkout and one using /test-checkout.

774 default_port = self.tool.port_factory.get()

775 self.tool.port_factory.get = lambda port_name=None: default_port

776

777 def test_modify_scm(self):

778 test_port = self.tool.port_factory.get('test')

779 self._write_test_file(test_port, 'another/test.html', "Dummy test conten ts")

780 self._write_test_file(test_port, 'platform/mac-snowleopard/another/test- expected.txt', "result A")

781 self._write_test_file(test_port, 'another/test-expected.txt', "result A" )

782

783 old_exact_matches = builders._exact_matches

784 try:

785 builders._exact_matches = {

786 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "spe cifiers": set(["mock-specifier"])},

787 }

788 OutputCapture().assert_outputs(self, self.command.execute, args=[

789 MockOptions(suffixes='txt', no_modify_scm=False, platform='test- mac-snowleopard'),

790 ['another/test.html'],

791 self.tool,

792 ], expected_stdout='{"add": [], "remove-lines": [], "delete": []}\n' )

793 finally:

794 builders._exact_matches = old_exact_matches

795

796 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(t est_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt')))

797 self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(te st_port.layout_tests_dir(), 'another/test-expected.txt')))

798

799 def test_no_modify_scm(self):

800 test_port = self.tool.port_factory.get('test')

801 self._write_test_file(test_port, 'another/test.html', "Dummy test conten ts")

802 self._write_test_file(test_port, 'platform/mac-snowleopard/another/test- expected.txt', "result A")

803 self._write_test_file(test_port, 'another/test-expected.txt', "result A" )

804

805 old_exact_matches = builders._exact_matches

806 try:

807 builders._exact_matches = {

808 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "spe cifiers": set(["mock-specifier"])},

809 }

810 OutputCapture().assert_outputs(self, self.command.execute, args=[

811 MockOptions(suffixes='txt', no_modify_scm=True, platform='test-m ac-snowleopard'),

812 ['another/test.html'],

813 self.tool,

814 ], expected_stdout='{"add": [], "remove-lines": [], "delete": ["/moc k-checkout/third_party/WebKit/tests/platform/mac-snowleopard/another/test-expect ed.txt"]}\n')

815 finally:

816 builders._exact_matches = old_exact_matches

817

818 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(t est_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt')))

819 self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(te st_port.layout_tests_dir(), 'another/test-expected.txt')))

820

821 def test_optimize_all_suffixes_by_default(self):

822 test_port = self.tool.port_factory.get('test')

823 self._write_test_file(test_port, 'another/test.html', "Dummy test conten ts")

824 self._write_test_file(test_port, 'platform/mac-snowleopard/another/test- expected.txt', "result A")

825 self._write_test_file(test_port, 'platform/mac-snowleopard/another/test- expected.png', "result A png")

826 self._write_test_file(test_port, 'another/test-expected.txt', "result A" )

827 self._write_test_file(test_port, 'another/test-expected.png', "result A png")

828

829 old_exact_matches = builders._exact_matches

830 try:

831 builders._exact_matches = {

832 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "spe cifiers": set(["mock-specifier"])},

833 }

834 oc = OutputCapture()

835 oc.capture_output()

836 self.command.execute(MockOptions(suffixes='txt,wav,png', no_modify_s cm=True, platform='test-mac-snowleopard'),

837 ['another/test.html'],

838 self.tool)

839 finally:

840 out, err, logs = oc.restore_output()

841 builders._exact_matches = old_exact_matches

842

843 self.assertEquals(out, '{"add": [], "remove-lines": [], "delete": ["/moc k-checkout/third_party/WebKit/tests/platform/mac-snowleopard/another/test-expect ed.txt", "/mock-checkout/third_party/WebKit/tests/platform/mac-snowleopard/anoth er/test-expected.png"]}\n')

844 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(t est_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt')))

845 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(t est_port.layout_tests_dir(), 'platform/mac/another/test-expected.png')))

846 self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(te st_port.layout_tests_dir(), 'another/test-expected.txt')))

847 self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(te st_port.layout_tests_dir(), 'another/test-expected.png')))

848

849

850 class TestAnalyzeBaselines(_BaseTestCase):

851 command_constructor = AnalyzeBaselines

852

853 def setUp(self):

854 super(TestAnalyzeBaselines, self).setUp()

855 self.port = self.tool.port_factory.get('test')

856 self.tool.port_factory.get = (lambda port_name=None, options=None: self. port)

857 self.lines = []

858 self.command._optimizer_class = _FakeOptimizer

859 self.command._write = (lambda msg: self.lines.append(msg)) # pylint bug warning about unnecessary lambda? pylint: disable=W0108

860

861 def test_default(self):

862 self.command.execute(MockOptions(suffixes='txt', missing=False, platform =None), ['passes/text.html'], self.tool)

863 self.assertEqual(self.lines,

864 ['passes/text-expected.txt:',

865 ' (generic): 123456'])

866

867 def test_missing_baselines(self):

868 self.command.execute(MockOptions(suffixes='png,txt', missing=True, platf orm=None), ['passes/text.html'], self.tool)

869 self.assertEqual(self.lines,

870 ['passes/text-expected.png: (no baselines found)',

871 'passes/text-expected.txt:',

872 ' (generic): 123456'])

873

874

875 class TestAutoRebaseline(_BaseTestCase):

876 command_constructor = AutoRebaseline

877

878 def _write_test_file(self, port, path, contents):

879 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path)

880 self.tool.filesystem.write_text_file(abs_path, contents)

881

882 def _setup_test_port(self):

883 test_port = self.tool.port_factory.get('test')

884 original_get = self.tool.port_factory.get

885

886 def get_test_port(port_name=None, options=None, **kwargs):

887 if not port_name:

888 return test_port

889 return original_get(port_name, options, **kwargs)

890 # Need to make sure all the ports grabbed use the test checkout path ins tead of the mock checkout path.

891 # FIXME: crbug.com/279494 - we shouldn't be doing this.

892 self.tool.port_factory.get = get_test_port

893

894 return test_port

895

896 def setUp(self):

897 super(TestAutoRebaseline, self).setUp()

898 self.command.latest_revision_processed_on_all_bots = lambda: 9000

899 self.command.bot_revision_data = lambda: [{"builder": "Mock builder", "r evision": "9000"}]

900

901 def test_release_builders(self):

902 old_exact_matches = builders._exact_matches

903 try:

904 builders._exact_matches = {

905 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

906 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "spe cifiers": set(["mock-specifier"])},

907 "MOCK Leopard ASAN": {"port_name": "test-mac-snowleopard", "spec ifiers": set(["mock-specifier"])},

908 }

909 self.assertEqual(self.command._release_builders(), ['MOCK Leopard'])

910 finally:

911 builders._exact_matches = old_exact_matches

912

913 def test_tests_to_rebaseline(self):

914 def blame(path):

915 return """

916 624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.ht ml [ ImageOnlyFailure ]

917 624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-04-28 04:52:41 +0000 13) Bug(foo) path/to/rebaseline-without-bug-number.ht ml [ NeedsRebaseline ]

918 624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/rebaseline-with -modifiers.html [ NeedsRebaseline ]

919 624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-04-28 04:52:41 +0000 12) crbug.com/24182 crbug.com/234 path/to/rebaseline- without-modifiers.html [ NeedsRebaseline ]

920 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/rebaseline-new-revision.h tml [ NeedsRebaseline ]

921 624caaaaaa path/to/TestExpectations (foo@chromium.org 2 013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.h tml [ NeedsRebaseline ]

922 0000000000 path/to/TestExpectations (foo@chromium.org 2 013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.htm l [ NeedsRebaseline ]

923 """

924 self.tool.scm().blame = blame

925

926 min_revision = 9000

927 self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revisio n, print_revisions=False), (

928 set(['path/to/rebaseline-without-bug-number.html', 'path/to/reba seline-with-modifiers.html', 'path/to/rebaseline-without-modifiers.html']),

929 5678,

930 'foobarbaz1@chromium.org',

931 set(['24182', '234']),

932 True))

933

934 def test_tests_to_rebaseline_over_limit(self):

935 def blame(path):

936 result = ""

937 for i in range(0, self.command.MAX_LINES_TO_REBASELINE + 1):

938 result += "624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) crbug.com/24182 path/t o/rebaseline-%s.html [ NeedsRebaseline ]\n" % i

939 return result

940 self.tool.scm().blame = blame

941

942 expected_list_of_tests = []

943 for i in range(0, self.command.MAX_LINES_TO_REBASELINE):

944 expected_list_of_tests.append("path/to/rebaseline-%s.html" % i)

945

946 min_revision = 9000

947 self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revisio n, print_revisions=False), (

948 set(expected_list_of_tests),

949 5678,

950 'foobarbaz1@chromium.org',

951 set(['24182']),

952 True))

953

954 def test_commit_message(self):

955 author = "foo@chromium.org"

956 revision = 1234

957 bugs = set()

958 self.assertEqual(self.command.commit_message(author, revision, bugs),

959 """Auto-rebaseline for r1234

960

961 http://src.chromium.org/viewvc/blink?view=revision&revision=1234

962

963 TBR=foo@chromium.org

964 """)

965

966 bugs = set(["234", "345"])

967 self.assertEqual(self.command.commit_message(author, revision, bugs),

968 """Auto-rebaseline for r1234

969

970 http://src.chromium.org/viewvc/blink?view=revision&revision=1234

971

972 BUG=234,345

973 TBR=foo@chromium.org

974 """)

975

976 def test_no_needs_rebaseline_lines(self):

977 def blame(path):

978 return """

979 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.ht ml [ ImageOnlyFailure ]

980 """

981 self.tool.scm().blame = blame

982

983 self.command.execute(MockOptions(optimize=True, verbose=False, move_over written_baselines=False, results_directory=False), [], self.tool)

984 self.assertEqual(self.tool.executive.calls, [])

985

986 def test_execute(self):

987 def blame(path):

988 return """

989 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-06-14 20:18:46 +0000 11) # Test NeedsRebaseline being in a comment doesn't bork parsing.

990 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.ht ml [ ImageOnlyFailure ]

991 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsReba seline ]

992 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-06-14 20:18:46 +0000 11) crbug.com/24182 [ SnowLeopard ] fast/dom/prototyp e-strawberry.html [ NeedsRebaseline ]

993 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-04-28 04:52:41 +0000 12) crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ]

994 624caaaaaa path/to/TestExpectations (foo@chromium.org 2 013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.h tml [ NeedsRebaseline ]

995 0000000000 path/to/TestExpectations (foo@chromium.org 2 013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.htm l [ NeedsRebaseline ]

996 """

997 self.tool.scm().blame = blame

998

999 test_port = self._setup_test_port()

1000

1001 old_builder_data = self.command.builder_data

1002

1003 def builder_data():

1004 old_builder_data()

1005 # have prototype-chocolate only fail on "MOCK Leopard".

1006 self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.r esults_from_string("""ADD_RESULTS({

1007 "tests": {

1008 "fast": {

1009 "dom": {

1010 "prototype-taco.html": {

1011 "expected": "PASS",

1012 "actual": "PASS TEXT",

1013 "is_unexpected": true

1014 },

1015 "prototype-chocolate.html": {

1016 "expected": "FAIL",

1017 "actual": "PASS"

1018 },

1019 "prototype-strawberry.html": {

1020 "expected": "PASS",

1021 "actual": "IMAGE PASS",

1022 "is_unexpected": true

1023 }

1024 }

1025 }

1026 }

1027 });""")

1028 return self.command._builder_data

1029

1030 self.command.builder_data = builder_data

1031

1032 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expe ctations_file(), """

1033 crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ]

1034 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]

1035 crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebase line ]

1036 crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ]

1037 crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]

1038 crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]

1039 """)

1040

1041 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")

1042 self._write_test_file(test_port, 'fast/dom/prototype-strawberry.html', " Dummy test contents")

1043 self._write_test_file(test_port, 'fast/dom/prototype-chocolate.html', "D ummy test contents")

1044

1045 self.tool.executive = MockLineRemovingExecutive()

1046

1047 old_exact_matches = builders._exact_matches

1048 try:

1049 builders._exact_matches = {

1050 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

1051 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "speci fiers": set(["mock-specifier"])},

1052 }

1053

1054 self.command.tree_status = lambda: 'closed'

1055 self.command.execute(MockOptions(optimize=True, verbose=False, move_ overwritten_baselines=False, results_directory=False), [], self.tool)

1056 self.assertEqual(self.tool.executive.calls, [])

1057

1058 self.command.tree_status = lambda: 'open'

1059 self.tool.executive.calls = []

1060 self.command.execute(MockOptions(optimize=True, verbose=False, move_ overwritten_baselines=False, results_directory=False), [], self.tool)

1061

1062 self.assertEqual(self.tool.executive.calls, [

1063 [

1064 ['python', 'echo', 'copy-existing-baselines-internal', '--su ffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-c hocolate.html'],

1065 ['python', 'echo', 'copy-existing-baselines-internal', '--su ffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-s trawberry.html'],

1066 ['python', 'echo', 'copy-existing-baselines-internal', '--su ffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco. html'],

1067 ['python', 'echo', 'copy-existing-baselines-internal', '--su ffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-t aco.html'],

1068 ],

1069 [

1070 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate .html'],

1071 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberr y.html'],

1072 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'],

1073 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html '],

1074 ],

1075 [

1076 ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'fast/dom/prototype-chocolate.html'],

1077 ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'png', 'fast/dom/prototype-strawberry.html'],

1078 ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html'],

1079 ],

1080 ['git', 'cl', 'upload', '-f'],

1081 ['git', 'pull'],

1082 ['git', 'cl', 'dcommit', '-f'],

1083 ['git', 'cl', 'set_close'],

1084 ])

1085

1086 # The mac ports should both be removed since they're the only ones i n builders._exact_matches.

1087 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_ to_generic_test_expectations_file()), """

1088 crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ]

1089 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]

1090 crbug.com/24182 [ Linux Win ] fast/dom/prototype-chocolate.html [ NeedsRebaselin e ]

1091 crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]

1092 crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]

1093 """)

1094 finally:

1095 builders._exact_matches = old_exact_matches

1096

1097 def test_execute_git_cl_hangs(self):

1098 def blame(path):

1099 return """

1100 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsReba seline ]

1101 """

1102 self.tool.scm().blame = blame

1103

1104 test_port = self._setup_test_port()

1105

1106 old_builder_data = self.command.builder_data

1107

1108 def builder_data():

1109 old_builder_data()

1110 # have prototype-chocolate only fail on "MOCK Leopard".

1111 self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.r esults_from_string("""ADD_RESULTS({

1112 "tests": {

1113 "fast": {

1114 "dom": {

1115 "prototype-taco.html": {

1116 "expected": "PASS",

1117 "actual": "PASS TEXT",

1118 "is_unexpected": true

1119 }

1120 }

1121 }

1122 }

1123 });""")

1124 return self.command._builder_data

1125

1126 self.command.builder_data = builder_data

1127

1128 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expe ctations_file(), """

1129 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]

1130 """)

1131

1132 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")

1133

1134 old_exact_matches = builders._exact_matches

1135 try:

1136 builders._exact_matches = {

1137 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "speci fiers": set(["mock-specifier"])},

1138 }

1139

1140 self.command.SECONDS_BEFORE_GIVING_UP = 0

1141 self.command.tree_status = lambda: 'open'

1142 self.tool.executive.calls = []

1143 self.command.execute(MockOptions(optimize=True, verbose=False, move_ overwritten_baselines=False, results_directory=False), [], self.tool)

1144

1145 self.assertEqual(self.tool.executive.calls, [

1146 [

1147 ['python', 'echo', 'copy-existing-baselines-internal', '--su ffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-t aco.html'],

1148 ],

1149 [

1150 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html '],

1151 ],

1152 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '-- suffixes', 'txt', 'fast/dom/prototype-taco.html']],

1153 ['git', 'cl', 'upload', '-f'],

1154 ])

1155 finally:

1156 builders._exact_matches = old_exact_matches

1157

1158 def test_execute_test_passes_everywhere(self):

1159 def blame(path):

1160 return """

1161 6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2 013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsReba seline ]

1162 """

1163 self.tool.scm().blame = blame

1164

1165 test_port = self._setup_test_port()

1166

1167 old_builder_data = self.command.builder_data

1168

1169 def builder_data():

1170 self.command._builder_data['MOCK Leopard'] = self.command._builder_d ata['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({

1171 "tests": {

1172 "fast": {

1173 "dom": {

1174 "prototype-taco.html": {

1175 "expected": "FAIL",

1176 "actual": "PASS",

1177 "is_unexpected": true

1178 }

1179 }

1180 }

1181 }

1182 });""")

1183 return self.command._builder_data

1184

1185 self.command.builder_data = builder_data

1186

1187 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expe ctations_file(), """

1188 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]

1189 """)

1190

1191 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")

1192

1193 self.tool.executive = MockLineRemovingExecutive()

1194

1195 old_exact_matches = builders._exact_matches

1196 try:

1197 builders._exact_matches = {

1198 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},

1199 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "speci fiers": set(["mock-specifier"])},

1200 }

1201

1202 self.command.tree_status = lambda: 'open'

1203 self.command.execute(MockOptions(optimize=True, verbose=False, move_ overwritten_baselines=False, results_directory=False), [], self.tool)

1204 self.assertEqual(self.tool.executive.calls, [

1205 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '-- suffixes', '', 'fast/dom/prototype-taco.html']],

1206 ['git', 'cl', 'upload', '-f'],

1207 ['git', 'pull'],

1208 ['git', 'cl', 'dcommit', '-f'],

1209 ['git', 'cl', 'set_close'],

1210 ])

1211

1212 # The mac ports should both be removed since they're the only ones i n builders._exact_matches.

1213 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_ to_generic_test_expectations_file()), """

1214 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]

1215 """)

1216 finally:

1217 builders._exact_matches = old_exact_matches

1218

1219

1220 class TestRebaselineOMatic(_BaseTestCase):

1221 command_constructor = RebaselineOMatic

1222

1223 def setUp(self):

1224 super(TestRebaselineOMatic, self).setUp()

1225 self._logs = []

1226

1227 def _mock_log_to_server(self, log=''):

1228 self._logs.append(log)

1229

1230 def test_run_logged_command(self):

1231 self.command._verbose = False

1232 self.command._post_log_to_server = self._mock_log_to_server

1233 self.command._run_logged_command(['echo', 'foo'])

1234 self.assertEqual(self.tool.executive.calls, [['echo', 'foo']])

1235 self.assertEqual(self._logs, ['MOCK STDOUT'])

1236

1237 def test_do_one_rebaseline(self):

1238 self.command._verbose = False

1239 self.command._post_log_to_server = self._mock_log_to_server

1240

1241 oc = OutputCapture()

1242 oc.capture_output()

1243 self.command._do_one_rebaseline()

1244 out, _, _ = oc.restore_output()

1245

1246 self.assertEqual(out, '')

1247 self.assertEqual(self.tool.executive.calls, [

1248 ['git', 'pull'],

1249 ['/mock-checkout/third_party/WebKit/tools/webkit-patch', 'auto-rebas eline'],

1250 ])

1251 self.assertEqual(self._logs, ['MOCK STDOUT'])

1252

1253 def test_do_one_rebaseline_verbose(self):

1254 self.command._verbose = True

1255 self.command._post_log_to_server = self._mock_log_to_server

1256

1257 oc = OutputCapture()

1258 oc.capture_output()

1259 self.command._do_one_rebaseline()

1260 out, _, _ = oc.restore_output()

1261

1262 self.assertEqual(out, 'MOCK STDOUT\n')

1263 self.assertEqual(self.tool.executive.calls, [

1264 ['git', 'pull'],

1265 ['/mock-checkout/third_party/WebKit/tools/webkit-patch', 'auto-rebas eline', '--verbose'],

1266 ])

1267 self.assertEqual(self._logs, ['MOCK STDOUT'])

OLD	NEW

« no previous file with comments | « sky/tools/webkitpy/tool/commands/rebaseline.py ('k') | sky/tools/webkitpy/tool/commands/stepsequence.py » ('j') | no next file with comments »