Some attempts to make PageLoadTest.Reliability faster and hopefully less flak...

chrome/test/reliability/page_load_test.cc

 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // This file provides reliablity test which runs under UI test framework. The
 // test is intended to run within QEMU environment.
 //
 // Usage 1: reliability_test
-// Upon invocation, it visits a hard coded list of URLs. This is mainly used
-// by buildbot, to verify reliability_test itself runs ok.
+// Upon invocation, it visits a hard coded list of sample pages. This is mainly
+// used by buildbot, to verify reliability_test itself runs ok.
 //
 // Usage 2: reliability_test --site=url --startpage=start --endpage=end [...]
 // Upon invocation, it visits a list of URLs constructed as
 // "http://url/page?id=k". (start <= k <= end).
 //
 // Usage 3: reliability_test --list=file --startline=start --endline=end [...]
 // Upon invocation, it visits each of the URLs on line numbers between start
 // and end, inclusive, stored in the input file. The line number starts from 1.
 //
 // If both "--site" and "--list" are provided, the "--site" set of arguments
 // are ignored.
 //
 // Optional Switches:
 // --iterations=num: goes through the list of URLs constructed in usage 2 or 3
 //                   num times.
 // --continuousload: continuously visits the list of URLs without restarting
 //                    browser for each page load.
 // --memoryusage: prints out memory usage when visiting each page.
 // --endurl=url: visits the specified url in the end.
 // --logfile=filepath: saves the visit log to the specified path.
 // --timeout=millisecond: time out as specified in millisecond during each
 //                        page load.
 // --nopagedown: won't simulate page down key presses after page load.
-// --savedebuglog: save Chrome and v8 debug log for each page loaded.
+// --savedebuglog: save Chrome, V8, and test debug log for each page loaded.
 
 #include <fstream>
 #include <iostream>
 
 #include "base/command_line.h"
+#include "base/file_path.h"
 #include "base/file_util.h"
 #include "base/path_service.h"
 #include "base/string_util.h"
 #include "base/time.h"
 #include "base/time_format.h"
 #include "chrome/browser/net/url_fixer_upper.h"
 #include "chrome/common/chrome_constants.h"
 #include "chrome/common/chrome_paths.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/common/logging_chrome.h"
@@ skipping 21 matching lines @@
 const wchar_t kIterationSwitch[] = L"iterations";
 const wchar_t kContinuousLoadSwitch[] = L"continuousload";
 const wchar_t kMemoryUsageSwitch[] = L"memoryusage";
 const wchar_t kEndURLSwitch[] = L"endurl";
 const wchar_t kLogFileSwitch[] = L"logfile";
 const wchar_t kTimeoutSwitch[] = L"timeout";
 const wchar_t kNoPageDownSwitch[] = L"nopagedown";
 const wchar_t kSaveDebugLogSwitch[] = L"savedebuglog";
 
 std::wstring server_url = L"http://urllist.com";
-const wchar_t test_url_1[] = L"http://www.google.com";
-const wchar_t test_url_2[] = L"about:crash";
-const wchar_t test_url_3[] = L"http://www.youtube.com";
+const wchar_t test_page_1[] = L"page1.html";
+const wchar_t test_page_2[] = L"page2.html";
+const wchar_t crash_url[] = L"about:crash";
 
 // These are copied from v8 definitions as we cannot include them.
 const wchar_t kV8LogFileSwitch[] = L"logfile";
 const wchar_t kV8LogFileDefaultName[] = L"v8.log";
 
 bool append_page_id = false;
 int32 start_page;
 int32 end_page;
 std::wstring url_file_path;
 int32 start_index = 1;
@@ skipping 75 matching lines @@
           // Page down twice.
           scoped_ptr<BrowserProxy> browser(automation()->GetBrowserWindow(0));
           if (browser.get()) {
             scoped_ptr<WindowProxy> window(browser->GetWindow());
             if (window.get()) {
               bool activation_timeout;
               browser->BringToFrontWithTimeout(action_max_timeout_ms(),
                                                &activation_timeout);
               if (!activation_timeout) {
                 window->SimulateOSKeyPress(VK_NEXT, 0);
                 Sleep(sleep_timeout_ms());

[Patrick Johnson, 2009/03/24 00:18:21]

What is the reason we need to wait 2 seconds after each page down key press? Can
this time be reduced?

[huanr, 2009/03/24 04:28:03]

Wait for page down to finish in VM environment which will be slow with full page
heap. I plan to make UI control synchronous.

                 window->SimulateOSKeyPress(VK_NEXT, 0);
                 Sleep(sleep_timeout_ms());
               }
             }
           }
         }
       }
     }
 
     if (!continuous_load) {
@@ skipping 102 matching lines @@
     } else {
       // Don't run if single process mode.
       // Also don't run if running as a standalone program which is for
       // distributed testing, to avoid mistakenly hitting web sites with many
       // instances.
       if (in_process_renderer() || stand_alone)
         return;
       // For usage 1
       NavigationMetrics metrics;
       if (timeout_ms == INFINITE)
-        timeout_ms = 30000;
+        timeout_ms = 2000;
 
-      NavigateToURLLogResult(GURL(test_url_1), log_file, &metrics);
+      // Paging down is unnecessary for usage 1.
+      page_down = false;

[Patrick Johnson, 2009/03/24 00:18:21]

I'm not completely happy with this, as this means the page down code path won't
be tested by usage 1. However, it does save time in the test (8 seconds).

+
+      FilePath sample_data_dir = GetSampleDataDir();
+      FilePath test_page_1 = sample_data_dir.AppendASCII("page1.html");
+      FilePath test_page_2 = sample_data_dir.AppendASCII("page2.html");
+
+      GURL test_url_1 = net::FilePathToFileURL(test_page_1);
+      GURL test_url_2 = net::FilePathToFileURL(test_page_2);
+
+      NavigateToURLLogResult(test_url_1, log_file, &metrics);
       // Verify everything is fine
       EXPECT_EQ(NAVIGATION_SUCCESS, metrics.result);
       EXPECT_EQ(0, metrics.crash_dump_count);
       EXPECT_EQ(true, metrics.browser_clean_exit);
       EXPECT_EQ(1, metrics.browser_launch_count);
       // Both starting page and test_url_1 are loaded.
       EXPECT_EQ(2, metrics.page_load_count);
       EXPECT_EQ(0, metrics.browser_crash_count);
       EXPECT_EQ(0, metrics.renderer_crash_count);
       EXPECT_EQ(0, metrics.plugin_crash_count);
 
       // Go to "about:crash"
-      uint32 crash_timeout_ms = timeout_ms / 2;
-      std::swap(timeout_ms, crash_timeout_ms);
-      NavigateToURLLogResult(GURL(test_url_2), log_file, &metrics);
-      std::swap(timeout_ms, crash_timeout_ms);
+      NavigateToURLLogResult(GURL(crash_url), log_file, &metrics);
       // Page load crashed and test automation timed out.
       EXPECT_EQ(NAVIGATION_TIME_OUT, metrics.result);
       // Found a crash dump
       EXPECT_EQ(1, metrics.crash_dump_count) << kFailedNoCrashService;
       // Browser did not crash, and exited cleanly.
       EXPECT_EQ(true, metrics.browser_clean_exit);
       EXPECT_EQ(1, metrics.browser_launch_count);
       // Only starting page was loaded.
       EXPECT_EQ(1, metrics.page_load_count);
       EXPECT_EQ(0, metrics.browser_crash_count);
       // Renderer crashed.
       EXPECT_EQ(1, metrics.renderer_crash_count);
       EXPECT_EQ(0, metrics.plugin_crash_count);
 
-      uint32 youtube_timeout_ms = timeout_ms * 2;
-      std::swap(timeout_ms, youtube_timeout_ms);
-      NavigateToURLLogResult(GURL(test_url_3), log_file, &metrics);
-      std::swap(timeout_ms, youtube_timeout_ms);
+      NavigateToURLLogResult(test_url_2, log_file, &metrics);
       // The data on previous crash should be cleared and we should get
       // metrics for a successful page load.
       EXPECT_EQ(NAVIGATION_SUCCESS, metrics.result);
       EXPECT_EQ(0, metrics.crash_dump_count);
       EXPECT_EQ(true, metrics.browser_clean_exit);
       EXPECT_EQ(1, metrics.browser_launch_count);
       EXPECT_EQ(0, metrics.browser_crash_count);
       EXPECT_EQ(0, metrics.renderer_crash_count);
       EXPECT_EQ(0, metrics.plugin_crash_count);
 
@@ skipping 185 matching lines @@
         local_state->GetInteger(prefs::kStabilityCrashCount);
     metrics->renderer_crash_count =
         local_state->GetInteger(prefs::kStabilityRendererCrashCount);
     // TODO(huanr)
     metrics->plugin_crash_count = 0;
 
     if (!metrics->browser_clean_exit)
       metrics->browser_crash_count++;
   }
 
+  FilePath GetSampleDataDir() {
+    FilePath test_dir;
+    PathService::Get(chrome::DIR_TEST_DATA, &test_dir);
+    test_dir = test_dir.AppendASCII("reliability");
+    test_dir = test_dir.AppendASCII("sample_pages");
+    return test_dir;
+  }
+
   // The pathname of Chrome's crash dumps directory.
   std::wstring crash_dumps_dir_path_;
 
   // The set of all the crash dumps we have seen.  Each crash generates a
   // .dmp and a .txt file in the crash dumps directory.  We only store the
   // .dmp files in this set.
   //
   // The set is implemented as a std::map.  The key is the file name, and
   // the value is false (the file is not in the set) or true (the file is
   // in the set).  The initial value for any key in std::map is 0 (false),
@@ skipping 106 matching lines @@
         if (v8_command_line.HasSwitch(kV8LogFileSwitch)) {
           v8_log_path = v8_command_line.GetSwitchValue(kV8LogFileSwitch);
           if (!file_util::AbsolutePath(&v8_log_path)) {
             v8_log_path.clear();
           }
         }
       }
     }
   }
 }