OLD | NEW |
| (Empty) |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 # | |
5 # This file lists the pyauto tests that run as a part of the functional test | |
6 # suite. | |
7 # | |
8 # Tests can be enabled on a per-platform basis. Tests not listed here will | |
9 # not be run. | |
10 # | |
11 # Within each platform, tests are specified in up to 3 groups: (1) enabled | |
12 # tests; (2) tests that are permanently disabled because they do not apply | |
13 # to the given platform and hence should not be run; and (3) disabled tests | |
14 # that need to be investigated/fixed. Tests are listed alphabetically within | |
15 # each group. | |
16 # | |
17 # Test names can be specified in any of the following ways: | |
18 # 1. as a module, in which case all tests in that module will be run | |
19 # example: 'test_basic' | |
20 # 2. or as a test case, in which case all tests in that test case will be run | |
21 # example: 'test_basic.SimpleTest' | |
22 # 3. or as an individual test | |
23 # example: 'test_basic.SimpleTest.testCanOpenGoogle' | |
24 # | |
25 # Tests beginning with '-' will be excluded. This can be used to enforce | |
26 # exclusions for a particular platform. | |
27 # Names beginning with '@' will be treated as suite names and will be | |
28 # recursively expanded. | |
29 | |
30 { | |
31 # This suite gets run on 'Google Chrome' builds. | |
32 'FULL': { | |
33 'all': [ | |
34 'about_plugins_ui.AboutPluginsUITest', | |
35 'about_plugins_ui.ChromeAboutPluginsUITest', | |
36 'apptest', | |
37 'autofill', | |
38 'codesign', | |
39 'crash_reporter', | |
40 'execute_javascript', | |
41 'extensions', | |
42 'fullscreen_mouselock', | |
43 'gtalk.test_basic', | |
44 'infobars', | |
45 'media.media_basic_playback', | |
46 'multiprofile', | |
47 'nacl_sdk', | |
48 'ntp', | |
49 'omnibox', | |
50 'passwords', | |
51 'prefs', | |
52 'prefs_ui', | |
53 'pyauto_webdriver', | |
54 'search_engines', | |
55 'special_tabs', | |
56 'test_pyauto', | |
57 | |
58 # =========================== | |
59 # Permanently-disabled tests. | |
60 # =========================== | |
61 | |
62 # ================================================== | |
63 # Disabled tests that need to be investigated/fixed. | |
64 # ================================================== | |
65 # crbug.com/145006 | |
66 '-about_plugins_ui.ChromeAboutPluginsUITest.testEnableAndDisableFlashPlugi
n', | |
67 # crbug.com/100365 | |
68 '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', | |
69 # crbug.com/171828 | |
70 '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs', | |
71 # The source is behind. Waiting for dev to automate the update. | |
72 # crbug.com/109160 | |
73 '-execute_javascript.ExecuteJavascriptTest.testExecuteJavascriptInExtensio
n', | |
74 # crbug.com/123396 | |
75 '-fullscreen_mouselock.FullscreenMouselockTest.testPatternsForFSAndML', | |
76 # crbug.com/132665 | |
77 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavBackToP
revPage', | |
78 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavToNewPa
ge', | |
79 '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavBackToPrev
Page', | |
80 '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavToNewPage'
, | |
81 # crbug.com/140460 | |
82 '-fullscreen_mouselock.FullscreenMouselockTest.testNoTabFSExitWhenJSExitMo
useLock', | |
83 # crosbug.com/136875 | |
84 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSDoesNotExitForAnch
orLinks', | |
85 # crbug.com/179263 | |
86 '-gtalk.test_basic.BasicTest.testCurrentVersion', | |
87 '-gtalk.test_basic.BasicTest.testRCVersion', | |
88 # crbug.com/165796 | |
89 '-infobars.OneClickInfobarTest', | |
90 # crbug.com/131874 | |
91 '-infobars.OneClickInfobarTest.testNoOneClickInfobarAfterCancel', | |
92 # crbug.com/133315 | |
93 '-infobars.OneClickInfobarTest.testDisplayOneClickInfobarAfterDismiss', | |
94 # Mysteriously broken? | |
95 # crbug.com/138857 | |
96 '-multiprofile.MultiprofileTest.testMakeSearchEngineDefaultInMultiprofile'
, | |
97 # crbug.com/179268 | |
98 '-ntp.NTPTest.testCannotUninstallWebStore', | |
99 '-ntp.NTPTest.testGetAppsInNewProfile', | |
100 '-ntp.NTPTest.testGetAppsWhenInstallApp', | |
101 '-ntp.NTPTest.testGetAppsWhenInstallNonApps', | |
102 '-ntp.NTPTest.testUninstallApp', | |
103 # crbug.com/143308 | |
104 '-omnibox.OmniboxLiveTest.testGoogleSearch', | |
105 # crbug.com/71715 | |
106 '-omnibox.OmniboxTest.testHistoryResult', | |
107 # crbug.com/123019 | |
108 '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', | |
109 # crbug.com/137041 | |
110 '-omnibox.OmniboxTest.testDifferentTypesOfResults', | |
111 # crbug.com/162341 | |
112 '-prefs.PrefsTest.testAllowSelectedGeoTracking', | |
113 '-prefs.PrefsTest.testDismissedInfobarSavesNoEntry', | |
114 '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', | |
115 '-prefs.PrefsTest.testGeolocationPref', | |
116 # crbug.com/85600 | |
117 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', | |
118 # crbug.com/157271 | |
119 '-prefs_ui.BasicSettingsUITest.testCancelStartupURLSetting', | |
120 '-prefs_ui.BasicSettingsUITest.testSetStartupPages', | |
121 '-prefs_ui.BasicSettingsUITest.testUseCurrentPagesForStartup', | |
122 # crbug.com/162341 | |
123 '-prefs_ui.PrefsUITest.testBehaviorValueCorrectlyDisplayed', | |
124 # crbug.com/112051 | |
125 '-prefs_ui.PrefsUITest.testChangeExceptionBehaviorUI', | |
126 # crbug.com/157271 | |
127 '-prefs_ui.PrefsUITest.testDeleteExceptionUI', | |
128 # crbug.com/151973 | |
129 '-prefs_ui.PrefsUITest.testInitialLineEntryInIncognitoUI', | |
130 # crbug.com/162341 | |
131 '-prefs_ui.PrefsUITest.testLocationSettingOptionsUI', | |
132 '-prefs_ui.PrefsUITest.testNoInitialLineEntryInUI', | |
133 # crbug.com/132285 | |
134 '-special_tabs.SpecialTabsTest.testSpecialURLTabs', | |
135 # crbug.com/168081 | |
136 '-ntp.NTPTest.testCloseOneTab', | |
137 ], | |
138 | |
139 'win': [ | |
140 'gpu', | |
141 # =========================== | |
142 # Permanently-disabled tests. | |
143 # =========================== | |
144 | |
145 # ================================================== | |
146 # Disabled tests that need to be investigated/fixed. | |
147 # ================================================== | |
148 # crbug.com/105948 | |
149 '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', | |
150 # crbug.com/111289 | |
151 '-extensions.ExtensionsTest.testAllowAccessFileURLs', | |
152 # crbug.com/113090 | |
153 '-extensions.ExtensionsTest.testAllowIncognitoExtension', | |
154 # crbug.com/171490 | |
155 '-passwords.PasswordTest.testPasswdInfoNotStoredWhenAutocompleteOff', | |
156 # crbug.com/117569 | |
157 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', | |
158 # crbug.com/98526 | |
159 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', | |
160 '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', | |
161 ], | |
162 | |
163 'mac': [ | |
164 # =========================== | |
165 # Permanently-disabled tests. | |
166 # =========================== | |
167 | |
168 # ================================================== | |
169 # Disabled tests that need to be investigated/fixed. | |
170 # ================================================== | |
171 # Keychain popups make autofill/password tests difficult: crbug.com/49378 | |
172 '-prefs_ui.PrefsUITest.testSetPasswordAndDelete', | |
173 # codesign tests should run *after* signing. crbug.com/50481 | |
174 '-codesign', | |
175 # crbug.com/124922 | |
176 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenBrowse
rLoseFocus', | |
177 # crbug.com/125989 | |
178 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenAlertD
ialogShow', | |
179 # Fails on chrome-mac-10_7-qa only: crbug.com/124886 | |
180 '-fullscreen_mouselock.FullscreenMouselockTest.testPrefsForFullscreenExit'
, | |
181 '-fullscreen_mouselock.FullscreenMouselockTest.testNoMouseLockWhenCancelFS
', | |
182 # crbug.com/121484 | |
183 '-multiprofile.MultiprofileTest.test20NewProfiles', | |
184 '-ntp.NTPTest.testLaunchAppNewWindow', # crbug.com/79812 | |
185 # crbug.com/70437 | |
186 '-omnibox.OmniboxTest.testHistoryResult', | |
187 # crbug.com/91617 | |
188 '-omnibox.OmniboxTest.testContentHistory', | |
189 # Keychain popups make autofill/password tests difficult: crbug.com/49378 | |
190 '-passwords', | |
191 # crbug.com/69619 | |
192 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', | |
193 # crbug.com/98526 | |
194 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', | |
195 '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', | |
196 ], | |
197 | |
198 'linux': [ | |
199 'test_clean_exit', | |
200 | |
201 # =========================== | |
202 # Permanently-disabled tests. | |
203 # =========================== | |
204 # System password manager obstructs password automation. | |
205 '-passwords', | |
206 | |
207 # ================================================== | |
208 # Disabled tests that need to be investigated/fixed. | |
209 # ================================================== | |
210 # crbug.com/111289 | |
211 '-extensions.ExtensionsTest.testAllowAccessFileURLs', | |
212 # crbug.com/91033 | |
213 '-omnibox.OmniboxTest.testOmniboxSearchHistory', | |
214 ], | |
215 | |
216 'chromeos': [ | |
217 'chromeos_basic', | |
218 'chromeos_browser', | |
219 'chromeos_crosh', | |
220 'chromeos_power', | |
221 'chromeos_prefs', | |
222 'chromeos_security', | |
223 'chromeos_time', | |
224 'secure_shell', | |
225 'youtube', | |
226 | |
227 # =========================== | |
228 # Permanently-disabled tests. | |
229 # =========================== | |
230 # No codesign verification on ChromeOS. | |
231 '-codesign', | |
232 # Sync is already signed in with the login account. | |
233 # So one-click infobar tests do tno apply to chromeos. | |
234 '-infobars.OneClickInfobarTest', | |
235 # Multi-profile doesn't apply to chromeos yet. | |
236 '-multiprofile', | |
237 '-ntp.NTPTest.testDifferentProfileNotAppearInMostVisited', | |
238 # No NaCl support on ChromeOS. | |
239 '-nacl_sdk', | |
240 | |
241 # ================================================== | |
242 # Disabled tests that need to be investigated/fixed. | |
243 # ================================================== | |
244 # crbug.com/132337 | |
245 '-autofill.AutofillTest.testTabOrderForEditAddress', | |
246 # crosbug.com/19556 | |
247 '-extensions.ExtensionsTest.testAllowAccessFileURLs', | |
248 '-extensions.ExtensionsTest.testAllowIncognitoExtension', | |
249 '-extensions.ExtensionsTest.testDisableEnableExtension', | |
250 # crbug.com/134593 | |
251 '-gtalk.test_basic.BasicTest.testCurrentVersion', | |
252 '-gtalk.test_basic.BasicTest.testRCVersion', | |
253 # crosbug.com/24496 | |
254 '-infobars.InfobarTest.testPluginCrashForMultiTabs', | |
255 # crbug.com/109035 | |
256 '-infobars.InfobarTest.testPluginCrashInfobar', | |
257 # crosbug.com/14256 | |
258 '-ntp.NTPTest.testLaunchAppFullScreen', | |
259 # Content history broken in omnibox. crosbug.com/14416 | |
260 '-omnibox.OmniboxTest.testContentHistory', | |
261 # crbug.com/91033 | |
262 '-omnibox.OmniboxTest.testOmniboxSearchHistory', | |
263 # crosbug.com/19760 | |
264 '-passwords.PasswordTest.testClearFetchedCredForNewUserName', | |
265 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', | |
266 # onunload popups get created in the same window on chromeos | |
267 # Session restore not working with PyAuto. crosbug.com/12648 | |
268 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', | |
269 '-prefs.PrefsTest.testSessionRestoreURLs', | |
270 '-prefs.PrefsTest.testSessionRestore', | |
271 # Deal with i18n chars. crosbug.com/12639 | |
272 '-omnibox.OmniboxTest.testCrazyFilenames', | |
273 # crosbug.com/20025 | |
274 '-chromeos_browser.ChromeosBrowserTest.testFullScreen', | |
275 # Chrome driver does not work in Chrome OS. | |
276 # crosbug.com/19556 | |
277 '-prefs_ui', | |
278 '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', | |
279 '-prefs.PrefsTest.testGeolocationPref', | |
280 '-pyauto_webdriver', | |
281 ], | |
282 }, | |
283 | |
284 # Performance tests. | |
285 'PERFORMANCE': { | |
286 'all': [ | |
287 'perf', | |
288 | |
289 # ================================================== | |
290 # Disabled tests that need to be investigated/fixed. | |
291 # ================================================== | |
292 '-perf.HTML5BenchmarkTest', # crbug.com/134476 | |
293 '-perf.LiveWebappLoadTest.testNewTabGmail', # crbug.com/136554 | |
294 '-perf.ScrollTest.testGmailScroll', # crbug.com/136554 | |
295 '-perf.WebGLTest.testWebGLField', # crbug.com/132797 | |
296 | |
297 # =========================== | |
298 # Permanently-disabled tests. | |
299 # =========================== | |
300 # Invoked outside of the pyauto_tests framework. | |
301 '-perf.BenchmarkPerfTest.testSpaceport', | |
302 '-perf.PopularSitesScrollTest.test2012Q3', | |
303 ], | |
304 'win': [ | |
305 # ================================================== | |
306 # Disabled tests that need to be investigated/fixed. | |
307 # ================================================== | |
308 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. | |
309 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 | |
310 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. | |
311 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. | |
312 | |
313 # =========================== | |
314 # Permanently-disabled tests. | |
315 # =========================== | |
316 '-perf.MemoryTest', # Designed only for ChromeOS. | |
317 '-perf.NetflixPerfTest', # Designed only for ChromeOS. | |
318 ], | |
319 'mac': [ | |
320 # ================================================== | |
321 # Disabled tests that need to be investigated/fixed. | |
322 # ================================================== | |
323 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. | |
324 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 | |
325 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. | |
326 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. | |
327 | |
328 # =========================== | |
329 # Permanently-disabled tests. | |
330 # =========================== | |
331 '-perf.MemoryTest', # Designed only for ChromeOS. | |
332 '-perf.NetflixPerfTest', # Designed only for ChromeOS. | |
333 ], | |
334 'linux': [ | |
335 # ================================================== | |
336 # Disabled tests that need to be investigated/fixed. | |
337 # ================================================== | |
338 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. | |
339 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 | |
340 '-perf.WebGLTest', # May not render WebGL; need to verify on QA bot. | |
341 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. | |
342 | |
343 # =========================== | |
344 # Permanently-disabled tests. | |
345 # =========================== | |
346 '-perf.MemoryTest', # Designed only for ChromeOS. | |
347 '-perf.NetflixPerfTest', # Designed only for ChromeOS. | |
348 ], | |
349 'chromeos': [ | |
350 # ================================================== | |
351 # Disabled tests that need to be investigated/fixed. | |
352 # ================================================== | |
353 '-perf.NetflixPerfTest', # crosbug.com/32320. | |
354 '-perf.YoutubePerfTest', # crbug.com/233706 | |
355 | |
356 # =========================== | |
357 # Permanently-disabled tests. | |
358 # =========================== | |
359 '-perf.GPUPerfTest', # Designed only for Chrome desktop. | |
360 ], | |
361 }, | |
362 | |
363 'PERFORMANCE_LAB': { | |
364 'all': [ | |
365 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', | |
366 'perf.FlashTest.testFlashGaming', | |
367 'perf.ScrollTest.testGooglePlusScroll', | |
368 'perf.TabPerfTest.test20Tabs', | |
369 'perf.WebGLTest.testWebGLSpaceRocks', | |
370 ], | |
371 }, | |
372 | |
373 'EMPTY': { | |
374 }, | |
375 | |
376 # ChromeOS flash tests. | |
377 'CHROMEOS_FLASH': { | |
378 'chromeos': [ | |
379 'flash', | |
380 ], | |
381 }, | |
382 | |
383 # ChromeOS volume tests. | |
384 'CHROMEOS_VOLUME': { | |
385 'chromeos': [ | |
386 'chromeos_volume', | |
387 ], | |
388 }, | |
389 | |
390 # ChromeOS Accessibility tests. | |
391 'CHROMEOS_ACCESSIBILITY': { | |
392 'chromeos': [ | |
393 'chromeos_accessibility', | |
394 ], | |
395 }, | |
396 | |
397 # PGO performance suite. | |
398 'PGO': { | |
399 'chromeos': [ | |
400 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', | |
401 'perf.PageCyclerTest', | |
402 'perf.ScrollTest.testGooglePlusScroll', | |
403 ], | |
404 }, | |
405 | |
406 # Subset of ChromeOS performance tests for seaboard. | |
407 'CHROMEOS_PERF_SEABOARD': { | |
408 'chromeos': [ | |
409 'perf.ScrollTest.testBlankPageScroll', | |
410 'perf.ScrollTest.testGooglePlusScroll', | |
411 'perf.ScrollTest.testTextScroll', | |
412 ], | |
413 }, | |
414 | |
415 # HTML5 media performance tests. | |
416 'AV_PERF': { | |
417 'linux': [ | |
418 'media.audio_latency_perf', | |
419 'media.audio_playback_perf', | |
420 'media.media_constrained_network_perf', | |
421 'media.media_scrub_perf', | |
422 'media.media_seek_perf', | |
423 'media.media_stat_perf', | |
424 'media.mixed_audio_latency_perf', | |
425 ], | |
426 'win': [ | |
427 'media.audio_latency_perf', | |
428 'media.audio_playback_perf', | |
429 'media.media_constrained_network_perf', | |
430 'media.media_scrub_perf', | |
431 'media.media_seek_perf', | |
432 'media.media_stat_perf', | |
433 'media.mixed_audio_latency_perf', | |
434 ], | |
435 }, | |
436 | |
437 # Trace event tests. | |
438 'TRACING': { | |
439 'all': [ | |
440 'tracing.tracing_smoke_test', | |
441 ], | |
442 }, | |
443 | |
444 # Chromoting tests. | |
445 'CHROMOTING': { | |
446 'all': [ | |
447 'chromoting.auth', | |
448 'chromoting.it2me_basic', | |
449 'chromoting.me2me_connect', | |
450 'chromoting.me2me_enable', | |
451 ], | |
452 'linux': [ | |
453 # TODO(yihongg): Me2me test against Linux is not working yet. | |
454 '-chromoting.me2me_connect', | |
455 '-chromoting.me2me_enable', | |
456 ], | |
457 'chromeos': [ | |
458 # ChromeOS doesn't yet support the chromoting host. | |
459 '-chromoting.auth', | |
460 '-chromoting.it2me_basic', | |
461 '-chromoting.me2me_connect', | |
462 '-chromoting.me2me_enable', | |
463 ], | |
464 }, | |
465 | |
466 # Pyauto functional tests running on coverage bots. | |
467 'CODE_COVERAGE': { | |
468 'all': [ | |
469 '@FULL', | |
470 ], | |
471 'linux': [ | |
472 # These tests fail on coverage bots. Disabling for now. | |
473 '-test_clean_exit', | |
474 '-about_plugins_ui', | |
475 '-autofill', | |
476 '-pyauto_webdriver', | |
477 '-extensions', | |
478 '-flash', | |
479 '-fullscreen_mouselock', | |
480 '-gtalk.test_basic', | |
481 '-infobars', | |
482 '-multiprofile', | |
483 '-prefs', | |
484 '-prefs_ui', | |
485 '-apptest', | |
486 '-plugins', | |
487 '-omnibox', | |
488 '-special_tabs', | |
489 '-ntp.NTPTest.testLaunchAppFullScreen', | |
490 '-media.media_basic_playback.MediaBasicPlaybackTest.testBasicPlaybackMatri
x', | |
491 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', | |
492 '-ntp.NTPTest.testUninstallApp', | |
493 '-ntp.NTPTest.testLaunchAppWithDefaultSettings', | |
494 '-ntp.NTPTest.testLaunchAppRegularTab', | |
495 '-ntp.NTPTest.testLaunchAppPinnedTab', | |
496 '-ntp.NTPTest.testGetAppsWhenInstallApp', | |
497 '-ntp.NTPTest.testLaunchAppNewWindow', | |
498 ], | |
499 }, | |
500 } | |
OLD | NEW |