OLD | NEW |
| (Empty) |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "base/environment.h" | |
6 #include "base/file_util.h" | |
7 #include "base/files/file_enumerator.h" | |
8 #include "base/path_service.h" | |
9 #include "base/strings/string_number_conversions.h" | |
10 #include "base/strings/string_split.h" | |
11 #include "base/strings/string_util.h" | |
12 #include "base/strings/stringprintf.h" | |
13 #include "base/strings/utf_string_conversions.h" | |
14 #include "base/sys_info.h" | |
15 #include "base/test/test_file_util.h" | |
16 #include "base/test/test_timeouts.h" | |
17 #include "base/time/time.h" | |
18 #include "chrome/common/chrome_constants.h" | |
19 #include "chrome/common/chrome_paths.h" | |
20 #include "chrome/common/chrome_switches.h" | |
21 #include "chrome/common/env_vars.h" | |
22 #include "chrome/test/automation/automation_proxy.h" | |
23 #include "chrome/test/base/test_switches.h" | |
24 #include "chrome/test/base/testing_profile.h" | |
25 #include "chrome/test/base/ui_test_utils.h" | |
26 #include "chrome/test/perf/perf_test.h" | |
27 #include "chrome/test/perf/perf_ui_test_suite.h" | |
28 #include "chrome/test/ui/ui_perf_test.h" | |
29 #include "content/public/common/content_switches.h" | |
30 #include "net/base/net_util.h" | |
31 #include "testing/perf/perf_test.h" | |
32 | |
33 using base::TimeDelta; | |
34 using base::TimeTicks; | |
35 | |
36 namespace { | |
37 | |
38 class StartupTest : public UIPerfTest { | |
39 public: | |
40 StartupTest() { | |
41 show_window_ = true; | |
42 } | |
43 virtual void SetUp() { | |
44 collect_profiling_stats_ = false; | |
45 } | |
46 virtual void TearDown() {} | |
47 | |
48 enum TestColdness { | |
49 WARM, | |
50 COLD | |
51 }; | |
52 | |
53 enum TestImportance { | |
54 NOT_IMPORTANT, | |
55 IMPORTANT | |
56 }; | |
57 | |
58 // Load a file on startup rather than about:blank. This tests a longer | |
59 // startup path, including resource loading. | |
60 void SetUpWithFileURL() { | |
61 const base::FilePath file_url = ui_test_utils::GetTestFilePath( | |
62 base::FilePath(base::FilePath::kCurrentDirectory), | |
63 base::FilePath(FILE_PATH_LITERAL("simple.html"))); | |
64 ASSERT_TRUE(base::PathExists(file_url)); | |
65 launch_arguments_.AppendArgPath(file_url); | |
66 } | |
67 | |
68 // Setup the command line arguments to capture profiling data for tasks. | |
69 void SetUpWithProfiling() { | |
70 profiling_file_ = ui_test_utils::GetTestFilePath( | |
71 base::FilePath(base::FilePath::kCurrentDirectory), | |
72 base::FilePath(FILE_PATH_LITERAL("task_profile.json"))); | |
73 launch_arguments_.AppendSwitchPath(switches::kProfilingOutputFile, | |
74 profiling_file_); | |
75 collect_profiling_stats_ = true; | |
76 } | |
77 | |
78 // Load a complex html file on startup represented by |which_tab|. | |
79 void SetUpWithComplexFileURL(unsigned int which_tab) { | |
80 const char* const kTestPageCyclerDomains[] = { | |
81 "www.google.com", "www.nytimes.com", | |
82 "www.yahoo.com", "espn.go.com", "www.amazon.com" | |
83 }; | |
84 unsigned int which_el = which_tab % arraysize(kTestPageCyclerDomains); | |
85 const char* this_domain = kTestPageCyclerDomains[which_el]; | |
86 | |
87 base::FilePath page_cycler_path; | |
88 PathService::Get(base::DIR_SOURCE_ROOT, &page_cycler_path); | |
89 page_cycler_path = page_cycler_path.AppendASCII("data") | |
90 .AppendASCII("page_cycler").AppendASCII("moz") | |
91 .AppendASCII(this_domain).AppendASCII("index.html"); | |
92 GURL file_url = net::FilePathToFileURL(page_cycler_path).Resolve("?skip"); | |
93 launch_arguments_.AppendArg(file_url.spec()); | |
94 } | |
95 | |
96 // Use the given profile in the test data extensions/profiles dir. This tests | |
97 // startup with extensions installed. | |
98 void SetUpWithExtensionsProfile(const char* profile) { | |
99 base::FilePath data_dir; | |
100 PathService::Get(chrome::DIR_TEST_DATA, &data_dir); | |
101 data_dir = data_dir.AppendASCII("extensions").AppendASCII("profiles"). | |
102 AppendASCII(profile); | |
103 set_template_user_data(data_dir); | |
104 } | |
105 | |
106 // Rewrite the preferences file to point to the proper image directory. | |
107 virtual void SetUpProfile() OVERRIDE { | |
108 UIPerfTest::SetUpProfile(); | |
109 if (profile_type_ != PerfUITestSuite::COMPLEX_THEME) | |
110 return; | |
111 | |
112 const base::FilePath pref_template_path(user_data_dir(). | |
113 AppendASCII(chrome::kInitialProfile). | |
114 AppendASCII("PreferencesTemplate")); | |
115 const base::FilePath pref_path(user_data_dir(). | |
116 AppendASCII(TestingProfile::kTestUserProfileDir). | |
117 Append(chrome::kPreferencesFilename)); | |
118 | |
119 // Read in preferences template. | |
120 std::string pref_string; | |
121 EXPECT_TRUE(base::ReadFileToString(pref_template_path, &pref_string)); | |
122 base::string16 format_string = base::ASCIIToUTF16(pref_string); | |
123 | |
124 // Make sure temp directory has the proper format for writing to prefs file. | |
125 #if defined(OS_POSIX) | |
126 std::wstring user_data_dir_w(base::ASCIIToWide(user_data_dir().value())); | |
127 #elif defined(OS_WIN) | |
128 std::wstring user_data_dir_w(user_data_dir().value()); | |
129 // In Windows, the FilePath will write '\' for the path separators; change | |
130 // these to a separator that won't trigger escapes. | |
131 std::replace(user_data_dir_w.begin(), | |
132 user_data_dir_w.end(), '\\', '/'); | |
133 #endif | |
134 | |
135 // Rewrite prefs file. | |
136 std::vector<base::string16> subst; | |
137 subst.push_back(base::WideToUTF16(user_data_dir_w)); | |
138 const std::string prefs_string = base::UTF16ToASCII( | |
139 ReplaceStringPlaceholders(format_string, subst, NULL)); | |
140 EXPECT_TRUE(base::WriteFile(pref_path, prefs_string.c_str(), | |
141 prefs_string.size())); | |
142 base::EvictFileFromSystemCache(pref_path); | |
143 } | |
144 | |
145 // Runs a test which loads |tab_count| tabs on startup, either as command line | |
146 // arguments or, if |restore_session| is true, by using session restore. | |
147 // |nth_timed_tab|, if non-zero, will measure time to load the first n+1 tabs. | |
148 void RunPerfTestWithManyTabs(const char* graph, const char* trace, | |
149 int tab_count, int nth_timed_tab, | |
150 bool restore_session); | |
151 | |
152 void RunStartupTest(const char* graph, const char* trace, | |
153 TestColdness test_cold, TestImportance test_importance, | |
154 PerfUITestSuite::ProfileType profile_type, | |
155 int num_tabs, int nth_timed_tab) { | |
156 bool important = (test_importance == IMPORTANT); | |
157 profile_type_ = profile_type; | |
158 | |
159 // Sets the profile data for the run. For now, this is only used for | |
160 // the non-default themes test. | |
161 if (profile_type != PerfUITestSuite::DEFAULT_THEME) { | |
162 set_template_user_data( | |
163 PerfUITestSuite::GetPathForProfileType(profile_type)); | |
164 } | |
165 | |
166 #if defined(NDEBUG) | |
167 const int kNumCyclesMax = 20; | |
168 #else | |
169 // Debug builds are too slow and we can't run that many cycles in a | |
170 // reasonable amount of time. | |
171 const int kNumCyclesMax = 10; | |
172 #endif | |
173 int numCycles = kNumCyclesMax; | |
174 scoped_ptr<base::Environment> env(base::Environment::Create()); | |
175 std::string numCyclesEnv; | |
176 if (env->GetVar(env_vars::kStartupTestsNumCycles, &numCyclesEnv) && | |
177 base::StringToInt(numCyclesEnv, &numCycles)) { | |
178 if (numCycles <= kNumCyclesMax) { | |
179 VLOG(1) << env_vars::kStartupTestsNumCycles | |
180 << " set in environment, so setting numCycles to " << numCycles; | |
181 } else { | |
182 VLOG(1) << env_vars::kStartupTestsNumCycles | |
183 << " is higher than the max, setting numCycles to " | |
184 << kNumCyclesMax; | |
185 numCycles = kNumCyclesMax; | |
186 } | |
187 } | |
188 | |
189 struct TimingInfo { | |
190 TimeDelta end_to_end; | |
191 float first_start_ms; | |
192 float last_stop_ms; | |
193 float first_stop_ms; | |
194 float nth_tab_stop_ms; | |
195 }; | |
196 TimingInfo timings[kNumCyclesMax]; | |
197 | |
198 CommandLine launch_arguments_without_trace_file(launch_arguments_); | |
199 for (int i = 0; i < numCycles; ++i) { | |
200 if (test_cold == COLD) { | |
201 base::FilePath dir_app; | |
202 ASSERT_TRUE(PathService::Get(chrome::DIR_APP, &dir_app)); | |
203 | |
204 base::FilePath chrome_exe(dir_app.Append(GetExecutablePath())); | |
205 ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_exe)); | |
206 #if defined(OS_WIN) | |
207 // chrome.dll is windows specific. | |
208 base::FilePath chrome_dll( | |
209 dir_app.Append(FILE_PATH_LITERAL("chrome.dll"))); | |
210 ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_dll)); | |
211 #endif | |
212 } | |
213 UITest::SetUp(); | |
214 TimeTicks end_time = TimeTicks::Now(); | |
215 | |
216 if (num_tabs > 0) { | |
217 float min_start; | |
218 float max_stop; | |
219 std::vector<float> times; | |
220 scoped_refptr<BrowserProxy> browser_proxy( | |
221 automation()->GetBrowserWindow(0)); | |
222 ASSERT_TRUE(browser_proxy.get()); | |
223 | |
224 if (browser_proxy->GetInitialLoadTimes( | |
225 TestTimeouts::action_max_timeout(), | |
226 &min_start, | |
227 &max_stop, | |
228 ×) && | |
229 !times.empty()) { | |
230 ASSERT_LT(nth_timed_tab, num_tabs); | |
231 ASSERT_EQ(times.size(), static_cast<size_t>(num_tabs)); | |
232 timings[i].first_start_ms = min_start; | |
233 timings[i].last_stop_ms = max_stop; | |
234 timings[i].first_stop_ms = times[0]; | |
235 timings[i].nth_tab_stop_ms = times[nth_timed_tab]; | |
236 } else { | |
237 // Browser might not support initial load times. | |
238 // Only use end-to-end time for this test. | |
239 num_tabs = 0; | |
240 } | |
241 } | |
242 timings[i].end_to_end = end_time - browser_launch_time(); | |
243 UITest::TearDown(); | |
244 | |
245 if (i == 0) { | |
246 // Re-use the profile data after first run so that the noise from | |
247 // creating databases doesn't impact all the runs. | |
248 clear_profile_ = false; | |
249 // Clear template_user_data_ so we don't try to copy it over each time | |
250 // through. | |
251 set_template_user_data(base::FilePath()); | |
252 } | |
253 } | |
254 | |
255 std::string times; | |
256 for (int i = 0; i < numCycles; ++i) { | |
257 base::StringAppendF(×, | |
258 "%.2f,", | |
259 timings[i].end_to_end.InMillisecondsF()); | |
260 } | |
261 perf_test::PrintResultList( | |
262 graph, std::string(), trace, times, "ms", important); | |
263 | |
264 if (num_tabs > 0) { | |
265 std::string name_base = trace; | |
266 std::string name; | |
267 | |
268 times.clear(); | |
269 name = name_base + "-start"; | |
270 for (int i = 0; i < numCycles; ++i) | |
271 base::StringAppendF(×, "%.2f,", timings[i].first_start_ms); | |
272 perf_test::PrintResultList( | |
273 graph, std::string(), name.c_str(), times, "ms", important); | |
274 | |
275 times.clear(); | |
276 name = name_base + "-first"; | |
277 for (int i = 0; i < numCycles; ++i) | |
278 base::StringAppendF(×, "%.2f,", timings[i].first_stop_ms); | |
279 perf_test::PrintResultList( | |
280 graph, std::string(), name.c_str(), times, "ms", important); | |
281 | |
282 if (nth_timed_tab > 0) { | |
283 // Display only the time necessary to load the first n tabs. | |
284 times.clear(); | |
285 name = name_base + "-" + base::IntToString(nth_timed_tab); | |
286 for (int i = 0; i < numCycles; ++i) | |
287 base::StringAppendF(×, "%.2f,", timings[i].nth_tab_stop_ms); | |
288 perf_test::PrintResultList( | |
289 graph, std::string(), name.c_str(), times, "ms", important); | |
290 } | |
291 | |
292 if (num_tabs > 1) { | |
293 // Display the time necessary to load all of the tabs. | |
294 times.clear(); | |
295 name = name_base + "-all"; | |
296 for (int i = 0; i < numCycles; ++i) | |
297 base::StringAppendF(×, "%.2f,", timings[i].last_stop_ms); | |
298 perf_test::PrintResultList( | |
299 graph, std::string(), name.c_str(), times, "ms", important); | |
300 } | |
301 } | |
302 } | |
303 | |
304 base::FilePath profiling_file_; | |
305 bool collect_profiling_stats_; | |
306 std::string trace_file_prefix_; | |
307 | |
308 // Are we using a profile with a complex theme? | |
309 PerfUITestSuite::ProfileType profile_type_; | |
310 }; | |
311 | |
312 TEST_F(StartupTest, PerfWarm) { | |
313 RunStartupTest("warm", "t", WARM, IMPORTANT, | |
314 PerfUITestSuite::DEFAULT_THEME, 0, 0); | |
315 } | |
316 | |
317 TEST_F(StartupTest, PerfReferenceWarm) { | |
318 UseReferenceBuild(); | |
319 RunStartupTest("warm", "t_ref", WARM, IMPORTANT, | |
320 PerfUITestSuite::DEFAULT_THEME, 0, 0); | |
321 } | |
322 | |
323 // TODO(mpcomplete): Should we have reference timings for all these? | |
324 | |
325 // dominich: Disabling as per http://crbug.com/100900. | |
326 #if defined(OS_WIN) | |
327 #define MAYBE_PerfCold DISABLED_PerfCold | |
328 #else | |
329 #define MAYBE_PerfCold PerfCold | |
330 #endif | |
331 | |
332 TEST_F(StartupTest, MAYBE_PerfCold) { | |
333 RunStartupTest("cold", "t", COLD, NOT_IMPORTANT, | |
334 PerfUITestSuite::DEFAULT_THEME, 0, 0); | |
335 } | |
336 | |
337 void StartupTest::RunPerfTestWithManyTabs(const char* graph, const char* trace, | |
338 int tab_count, int nth_timed_tab, | |
339 bool restore_session) { | |
340 // Initialize session with |tab_count| tabs. | |
341 for (int i = 0; i < tab_count; ++i) | |
342 SetUpWithComplexFileURL(i); | |
343 | |
344 if (restore_session) { | |
345 // Start the browser with these urls so we can save the session and exit. | |
346 UITest::SetUp(); | |
347 // Set flags to ensure profile is saved and can be restored. | |
348 #if defined(OS_CHROMEOS) || defined(OS_MACOSX) | |
349 set_shutdown_type(ProxyLauncher::USER_QUIT); | |
350 #endif | |
351 clear_profile_ = false; | |
352 // Quit and set flags to restore session. | |
353 UITest::TearDown(); | |
354 | |
355 // Clear all arguments for session restore, or the number of open tabs | |
356 // will grow with each restore. | |
357 CommandLine new_launch_arguments = CommandLine(CommandLine::NO_PROGRAM); | |
358 // Keep the branding switch if using a reference build. | |
359 if (launch_arguments_.HasSwitch(switches::kEnableChromiumBranding)) { | |
360 new_launch_arguments.AppendSwitch(switches::kEnableChromiumBranding); | |
361 } | |
362 // The session will be restored once per cycle for numCycles test cycles, | |
363 // and each time, UITest::SetUp will wait for |tab_count| tabs to | |
364 // finish loading. | |
365 new_launch_arguments.AppendSwitchASCII(switches::kRestoreLastSession, | |
366 base::IntToString(tab_count)); | |
367 launch_arguments_ = new_launch_arguments; | |
368 } | |
369 RunStartupTest(graph, trace, WARM, NOT_IMPORTANT, | |
370 PerfUITestSuite::DEFAULT_THEME, tab_count, nth_timed_tab); | |
371 } | |
372 | |
373 // http://crbug.com/101591 | |
374 #if defined(OS_WIN) && !defined(NDEBUG) | |
375 #define MAYBE_PerfFewTabs DISABLED_PerfFewTabs | |
376 #else | |
377 #define MAYBE_PerfFewTabs PerfFewTabs | |
378 #endif | |
379 | |
380 TEST_F(StartupTest, MAYBE_PerfFewTabs) { | |
381 RunPerfTestWithManyTabs("few_tabs", "cmdline", 5, 2, false); | |
382 } | |
383 | |
384 TEST_F(StartupTest, PerfFewTabsReference) { | |
385 UseReferenceBuild(); | |
386 RunPerfTestWithManyTabs("few_tabs", "cmdline-ref", 5, 2, false); | |
387 } | |
388 | |
389 TEST_F(StartupTest, PerfRestoreFewTabs) { | |
390 RunPerfTestWithManyTabs("few_tabs", "restore", 5, 2, true); | |
391 } | |
392 | |
393 TEST_F(StartupTest, PerfRestoreFewTabsReference) { | |
394 UseReferenceBuild(); | |
395 RunPerfTestWithManyTabs("few_tabs", "restore-ref", 5, 2, true); | |
396 } | |
397 | |
398 #if defined(OS_MACOSX) | |
399 // http://crbug.com/46609 | |
400 #define MAYBE_PerfSeveralTabsReference DISABLED_PerfSeveralTabsReference | |
401 #define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs | |
402 // http://crbug.com/52858 | |
403 #define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs | |
404 #define MAYBE_PerfExtensionContentScript50 DISABLED_PerfExtensionContentScript50 | |
405 #elif defined(OS_WIN) | |
406 // http://crbug.com/46609 | |
407 #if !defined(NDEBUG) | |
408 // This test is disabled on Windows Debug. See bug http://crbug.com/132279 | |
409 #define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs | |
410 #else | |
411 #define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs | |
412 #endif // !defined(NDEBUG) | |
413 // http://crbug.com/102584 | |
414 #define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs | |
415 #define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference | |
416 #define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50 | |
417 #else | |
418 #define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference | |
419 #define MAYBE_PerfSeveralTabs PerfSeveralTabs | |
420 #define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs | |
421 #define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50 | |
422 #endif | |
423 | |
424 // http://crbug.com/99604 | |
425 #if defined(OS_WIN) && !defined(NDEBUG) | |
426 #define MAYBE_PerfComplexTheme DISABLED_PerfComplexTheme | |
427 #else | |
428 #define MAYBE_PerfComplexTheme PerfComplexTheme | |
429 #endif | |
430 | |
431 TEST_F(StartupTest, MAYBE_PerfSeveralTabs) { | |
432 RunPerfTestWithManyTabs("several_tabs", "cmdline", 10, 4, false); | |
433 } | |
434 | |
435 TEST_F(StartupTest, MAYBE_PerfSeveralTabsReference) { | |
436 UseReferenceBuild(); | |
437 RunPerfTestWithManyTabs("several_tabs", "cmdline-ref", 10, 4, false); | |
438 } | |
439 | |
440 TEST_F(StartupTest, MAYBE_PerfRestoreSeveralTabs) { | |
441 RunPerfTestWithManyTabs("several_tabs", "restore", 10, 4, true); | |
442 } | |
443 | |
444 TEST_F(StartupTest, PerfRestoreSeveralTabsReference) { | |
445 UseReferenceBuild(); | |
446 RunPerfTestWithManyTabs("several_tabs", "restore-ref", 10, 4, true); | |
447 } | |
448 | |
449 TEST_F(StartupTest, PerfExtensionEmpty) { | |
450 SetUpWithFileURL(); | |
451 SetUpWithExtensionsProfile("empty"); | |
452 RunStartupTest("warm", "extension_empty", WARM, NOT_IMPORTANT, | |
453 PerfUITestSuite::DEFAULT_THEME, 1, 0); | |
454 } | |
455 | |
456 TEST_F(StartupTest, PerfExtensionContentScript1) { | |
457 SetUpWithFileURL(); | |
458 SetUpWithExtensionsProfile("content_scripts1"); | |
459 RunStartupTest("warm", "extension_content_scripts1", WARM, NOT_IMPORTANT, | |
460 PerfUITestSuite::DEFAULT_THEME, 1, 0); | |
461 } | |
462 | |
463 TEST_F(StartupTest, MAYBE_PerfExtensionContentScript50) { | |
464 SetUpWithFileURL(); | |
465 SetUpWithExtensionsProfile("content_scripts50"); | |
466 RunStartupTest("warm", "extension_content_scripts50", WARM, NOT_IMPORTANT, | |
467 PerfUITestSuite::DEFAULT_THEME, 1, 0); | |
468 } | |
469 | |
470 TEST_F(StartupTest, MAYBE_PerfComplexTheme) { | |
471 RunStartupTest("warm", "t-theme", WARM, NOT_IMPORTANT, | |
472 PerfUITestSuite::COMPLEX_THEME, 0, 0); | |
473 } | |
474 | |
475 TEST_F(StartupTest, ProfilingScript1) { | |
476 SetUpWithFileURL(); | |
477 SetUpWithProfiling(); | |
478 RunStartupTest("warm", "profiling_scripts1", WARM, NOT_IMPORTANT, | |
479 PerfUITestSuite::DEFAULT_THEME, 1, 0); | |
480 } | |
481 | |
482 } // namespace | |
OLD | NEW |