| OLD | NEW |
| (Empty) |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #ifndef CHROME_TEST_RELIABILITY_AUTOMATED_UI_TESTS_H_ | |
| 6 #define CHROME_TEST_RELIABILITY_AUTOMATED_UI_TESTS_H_ | |
| 7 | |
| 8 // This takes an input file of commands, which consist of a series of | |
| 9 // actions, and runs every command, reporting the status of each one | |
| 10 // to an output file once all the commands have been run. | |
| 11 // | |
| 12 // The input file should be an XML file that has a root of any name followed | |
| 13 // by a series of elements named "command" with child elements representing | |
| 14 // the various actions, in order, to be performed during each command. A | |
| 15 // command element can optionally include an "number" attribute to identify it. | |
| 16 // | |
| 17 // Example: | |
| 18 // <CommandList> | |
| 19 // <command number="1"><NewTab/><Navigate/><OpenWindow/><Navigate/><Back/> | |
| 20 // </command> | |
| 21 // <command number="2"><NewTab/><Navigate/><Navigate/><Back/><Forward/> | |
| 22 // </command> | |
| 23 // <command number="3"><CloseTab/><OpenWindow/><NewTab/><Navigate/><CloseTab/> | |
| 24 // </command> | |
| 25 // </CommandList> | |
| 26 // | |
| 27 // When the test is finished it will output results to the output file, | |
| 28 // overwriting any previous version of this file. The output file is an | |
| 29 // XML file which reports on each command, indicating whether it successfully | |
| 30 // ran and if there were any errors. | |
| 31 // | |
| 32 // Example: (actual output will probably contain more actions per command): | |
| 33 // <Report> | |
| 34 // <Executed command_number="1"><NewTab/><Navigate/><result><success/> | |
| 35 // </result> </Executed> | |
| 36 // <Executed command_number="2"><Back/><Forward/><result><success/></result> | |
| 37 // </Executed> | |
| 38 // <Executed command_number="3"><CloseTab/><result> | |
| 39 // <crash crash_dump="C:\a_crash.txt" command_completed="no"/></result> | |
| 40 // </Executed> | |
| 41 // </Report> | |
| 42 // | |
| 43 // A "crash" result will have two attributes, crash_dump, which points | |
| 44 // to the full path of crash dump associated with this crash, and | |
| 45 // command_completed which indicates whether or not the last | |
| 46 // action recorded was the final action of the command. | |
| 47 // | |
| 48 // Furthermore, each individual action may contain additional attributes | |
| 49 // to log non-fatal failures. If the attribute 'failed_to_complete="yes"' | |
| 50 // is present, then the action did not complete. If that attribute is present, | |
| 51 // an info, warning, or error attribute will also be present, and will contain | |
| 52 // a string describing the error. The presence of info means the failure was | |
| 53 // expected, probably due to a state making the action impossible to perform | |
| 54 // like trying to close the last remaining window. Warnings usually mean the | |
| 55 // action couldn't complete for an unknown and unexpected reason, but that the | |
| 56 // test state is probably fine. Errors are like warnings, but they mean the test | |
| 57 // state is probably incorrect, and more failures are likely to be caused by | |
| 58 // the same problem. | |
| 59 // | |
| 60 // Example of some failed actions: | |
| 61 // <CloseTab failed_to_complete="yes" info="would_have_exited_application"/> | |
| 62 // <Reload failed_to_complete="yes" warning="failed_to_apply_accelerator"/> | |
| 63 // <Star failed_to_complete="yes" error="browser_window_not_found"/> | |
| 64 // | |
| 65 // | |
| 66 // Switches | |
| 67 // --input : Specifies the input file, must be an absolute directory. | |
| 68 // Default is "C:\automated_ui_tests.txt" | |
| 69 // | |
| 70 // --output : Specifies the output file, must be an absolute directory. | |
| 71 // Default is "C:\automated_ui_tests_error_report.txt" | |
| 72 // | |
| 73 // | |
| 74 // Test reproduction options: | |
| 75 // | |
| 76 // If you're trying to reproduce the results from crash reports use the | |
| 77 // following switches | |
| 78 // | |
| 79 // --key : Specifies, via a comma delimited list, what actions to run. Examples: | |
| 80 // --key=SetUp,ZoomPlus,Forward,History,Navigate,Back,TearDown | |
| 81 // --key=SetUp,ZoomPlus | |
| 82 // Note, the second key doesn't include a TearDown, that will | |
| 83 // automatically be added if the result doesn't crash. | |
| 84 // | |
| 85 // --num-reproductions : Specifies the number of reproductions to run, the | |
| 86 // default is 1. Suggested use: run without this flag | |
| 87 // to see if we reproduce a crash, then run with the flag | |
| 88 // if there isn't a crash, to see if it might be a rare | |
| 89 // race condition that causes the crash. | |
| 90 // | |
| 91 // | |
| 92 // Debugging options: | |
| 93 // | |
| 94 // --debug : Will append each action that is performed to the output file, as | |
| 95 // soon as the action is performed. If the program finishes, this file | |
| 96 // will be overwritten with the normal results. This flag is used to | |
| 97 // help debug the tests if they are crashing before they get a chance | |
| 98 // to write their results to file. | |
| 99 // | |
| 100 // --wait-after-action : waits the specified amount of time (1s by default) | |
| 101 // after each action. Useful for debugging. | |
| 102 | |
| 103 #include <string> | |
| 104 | |
| 105 #include "chrome/test/reliability/automated_ui_test_base.h" | |
| 106 #include "chrome/test/ui/ui_test.h" | |
| 107 #include "third_party/libxml/chromium/libxml_utils.h" | |
| 108 #include "ui/events/keycodes/keyboard_codes.h" | |
| 109 | |
| 110 namespace base { | |
| 111 class Time; | |
| 112 } | |
| 113 | |
| 114 class AutomatedUITest : public AutomatedUITestBase { | |
| 115 protected: | |
| 116 AutomatedUITest(); | |
| 117 virtual ~AutomatedUITest(); | |
| 118 | |
| 119 // Runs a reproduction of one set of actions, reporting whether they crash | |
| 120 // or not. | |
| 121 void RunReproduction(); | |
| 122 | |
| 123 // Runs automated UI tests which are read from the input file. | |
| 124 // Reports crashes to the output file. | |
| 125 void RunAutomatedUITest(); | |
| 126 | |
| 127 // Attempts to perform an action based on the input string. See below for | |
| 128 // possible actions. Returns true if the action completes, false otherwise. | |
| 129 bool DoAction(const std::string& action); | |
| 130 | |
| 131 // Actions ------------------------------------------------------------------ | |
| 132 | |
| 133 // NOTE: This list is sorted alphabetically, so that we can easily detect | |
| 134 // missing actions. | |
| 135 | |
| 136 // Changes the encoding of the page (the encoding is selected at random | |
| 137 // from a list of encodings). | |
| 138 // Returns true if call to activate the accelerator is successful. | |
| 139 // XML element: <ChangeEncoding/> | |
| 140 bool ChangeEncoding(); | |
| 141 | |
| 142 // Opens one of the dialogs (chosen randomly) and exercises it. | |
| 143 // XML element: <Dialog/> | |
| 144 bool ExerciseDialog(); | |
| 145 | |
| 146 // Opens the JavaScriptConsole window. While it isn't modal, it takes focus | |
| 147 // from the current browser window, so most of the test can't continue until | |
| 148 // it is dismissed. | |
| 149 // XML element: <JavaScriptConsole/> | |
| 150 bool JavaScriptConsole(); | |
| 151 | |
| 152 // Opens the About dialog. This dialog is modal so a majority of the test | |
| 153 // can't be completed until it is dismissed. | |
| 154 // XML element: <About/> | |
| 155 bool OpenAboutDialog(); | |
| 156 | |
| 157 // Opens the Clear Browsing Data dialog, this dialog is modal so a majority of | |
| 158 // the test can't be completed until it is dismissed. | |
| 159 // XML element: <ClearBrowsingData/> | |
| 160 bool OpenClearBrowsingDataDialog(); | |
| 161 | |
| 162 // Opens the Search Engines dialog. While it isn't modal, it takes focus from | |
| 163 // the current browser window, so most of the test can't continue until it is | |
| 164 // dismissed. | |
| 165 // XML element: <EditSearchEngines/> | |
| 166 bool OpenEditSearchEnginesDialog(); | |
| 167 | |
| 168 // Opens the Import Settings dialog, this dialog is modal so a majority of | |
| 169 // the test can't be completed until it is dismissed. | |
| 170 // XML element: <ImportSettings/> | |
| 171 bool OpenImportSettingsDialog(); | |
| 172 | |
| 173 // Opens the Task Manager dialog. While it isn't modal, it takes focus from | |
| 174 // the current browser window, so most of the test can't continue until it is | |
| 175 // dismissed. | |
| 176 // XML element: <TaskManager/> | |
| 177 bool OpenTaskManagerDialog(); | |
| 178 | |
| 179 // Opens the View Passwords dialog, this dialog is modal so a majority of | |
| 180 // the test can't be completed until it is dismissed. | |
| 181 // XML element: <ViewPasswords/> | |
| 182 bool OpenViewPasswordsDialog(); | |
| 183 | |
| 184 // Opens the Options dialog. While it isn't modal, it takes focus from | |
| 185 // the current browser window, so most of the test can't continue until it is | |
| 186 // dismissed. | |
| 187 // XML element: <Options/> | |
| 188 bool Options(); | |
| 189 | |
| 190 // Stars the current page. This opens a dialog that may or may not be | |
| 191 // dismissed. | |
| 192 // XML element: <Star/> | |
| 193 bool StarPage(); | |
| 194 | |
| 195 // Views source of the current page. | |
| 196 // Returns true if call to activate the accelerator is successful. | |
| 197 // XML element: <ViewSource/> | |
| 198 bool ViewSource(); | |
| 199 | |
| 200 // Decreases the text size on the current active tab. | |
| 201 // XML element: <ZoomMinus/> | |
| 202 bool ZoomMinus(); | |
| 203 | |
| 204 // Increases the text size on the current active tab. | |
| 205 // XML element: <ZoomPlus/> | |
| 206 bool ZoomPlus(); | |
| 207 | |
| 208 // Test Dialog Actions ****************************************************** | |
| 209 // These are a special set of actions that perform multiple actions on a | |
| 210 // specified dialog. They run kTestDialogActionsToRun actions randomly | |
| 211 // chosen from test_dialog_possible_actions_ after opening the dialog. They | |
| 212 // then always end with a PressEscapeKey action, to attempt to close the | |
| 213 // dialog. | |
| 214 // | |
| 215 // The randomly performed actions are logged as child elements of the | |
| 216 // TestDialog action. For example (for kTestDialogActionsToRun = 4): | |
| 217 // <TestEditKeywords> <PressTabKey/><PressEnterKey/><DownArrow/> | |
| 218 // <DownArrow/><PressEscapeKey/> </TestEditKeywords> | |
| 219 | |
| 220 // Opens About dialog and runs random actions on it. | |
| 221 // XML element: <TestAboutChrome/> | |
| 222 bool TestAboutChrome(); | |
| 223 | |
| 224 // Opens Clear Browsing Data dialog and runs random actions on it. | |
| 225 // XML element: <TestClearBrowsingData/> | |
| 226 bool TestClearBrowsingData(); | |
| 227 | |
| 228 // Opens Edit Keywords dialog and runs random actions on it. | |
| 229 // XML element: <TestEditSearchEngines/> | |
| 230 bool TestEditSearchEngines(); | |
| 231 | |
| 232 // Opens Import Settings dialog and runs random actions on it. | |
| 233 // XML element: <TestImportSettings/> | |
| 234 bool TestImportSettings(); | |
| 235 | |
| 236 // Opens Options dialog and runs random actions on it. | |
| 237 // XML element: <TestOptions/> | |
| 238 bool TestOptions(); | |
| 239 | |
| 240 // Opens Task Manager and runs random actions on it. | |
| 241 // This has the possibility of killing both the browser and renderer | |
| 242 // processes, which will cause non-fatal errors for the remaining actions | |
| 243 // in this command. | |
| 244 // XML element: <TestTaskManager/> | |
| 245 bool TestTaskManager(); | |
| 246 | |
| 247 // Opens View Passwords dialog and runs random actions on it. | |
| 248 // XML element: <TestViewPasswords/> | |
| 249 bool TestViewPasswords(); | |
| 250 | |
| 251 // End Test Dialog Actions ************************************************** | |
| 252 | |
| 253 // Runs a limited set of actions designed to test dialogs. Will run | |
| 254 // |num_actions| from the set defined in test_dialog_possible_actions_. | |
| 255 bool FuzzyTestDialog(int num_actions); | |
| 256 | |
| 257 // Navigates to about:crash. | |
| 258 // XML element: <Crash/> | |
| 259 bool ForceCrash(); | |
| 260 | |
| 261 // Utility functions -------------------------------------------------------- | |
| 262 | |
| 263 // Opens init file, reads it into the reader, and closes the file. | |
| 264 // Returns false if there are any errors. | |
| 265 bool InitXMLReader(); | |
| 266 | |
| 267 // Closes the xml_writer and outputs the contents of its buffer to | |
| 268 // the output file. | |
| 269 bool WriteReportToFile(); | |
| 270 | |
| 271 // Appends the provided string to the output file. | |
| 272 void AppendToOutputFile(const std::string& append_string); | |
| 273 | |
| 274 // Logs a crash to the xml_writer in the form of: | |
| 275 // <result><crash crash_dump="|crash_dump|" command_completed="yes/no"/> | |
| 276 // </result> | |
| 277 // crash_dump - Location of crash dump if applicable. | |
| 278 // command_completed - True if all actions in the command were completed | |
| 279 // before the crash occured. | |
| 280 void LogCrashResult(const base::FilePath& crash_dump, | |
| 281 bool command_completed); | |
| 282 | |
| 283 // Logs a successful command to the xml_writer in the form of: | |
| 284 // <result><success/><result/> | |
| 285 void LogSuccessResult(); | |
| 286 | |
| 287 // Adds the attribute "reason=|reason|" to the current element. | |
| 288 // Used to log the reason for a given failure while performing an action. | |
| 289 void LogActionFailureReason(const std::string& reason); | |
| 290 | |
| 291 // Adds the attribute 'info="|info|"' to the current element. Used when an | |
| 292 // action could not complete for a non-serious issue. Usually because the | |
| 293 // state of the test wouldn't allow for a particular action. | |
| 294 void AddInfoAttribute(const std::string& info); | |
| 295 | |
| 296 // Adds the attribute "warning=|warning|" to the current element. Used when | |
| 297 // an action could not complete because of a potentially troublesome issue. | |
| 298 void AddWarningAttribute(const std::string& warning); | |
| 299 | |
| 300 // Adds the attribute "error=|error|" to the current element. Used when an | |
| 301 // action could not complete due to an unexpected problem which might | |
| 302 // invalidate the results of the entire command (not just the action). | |
| 303 // This is usually used when the testing environment isn't acting as we'd | |
| 304 // expect. For example, no chrome windows are focused, or key presses aren't | |
| 305 // being registered. | |
| 306 void AddErrorAttribute(const std::string& error); | |
| 307 | |
| 308 // Returns the full path of the crash dump. This is likely to be the | |
| 309 // .txt file, not the actual crash dump. Although they do share | |
| 310 // a common name. | |
| 311 base::FilePath GetMostRecentCrashDump(); | |
| 312 | |
| 313 // Returns true if the test has produced any new crash logs. | |
| 314 // A "new" crash log is one that was produced since DidCrash was last called | |
| 315 // with |update_total_crashes| set to true. | |
| 316 bool DidCrash(bool update_total_crashes); | |
| 317 | |
| 318 // Override the message logging in AutomatedUITestBase. | |
| 319 virtual void LogErrorMessage(const std::string& error) OVERRIDE; | |
| 320 virtual void LogWarningMessage(const std::string& warning) OVERRIDE; | |
| 321 virtual void LogInfoMessage(const std::string& info) OVERRIDE; | |
| 322 | |
| 323 // Overridden so that UI Test doesn't set up when the tests start. | |
| 324 // We use DoAction("SetUp") to set up, because it logs it and makes | |
| 325 // it easier to check for crashes when we start the browser. | |
| 326 virtual void SetUp() OVERRIDE {} | |
| 327 | |
| 328 // Overridden so that UI Test doesn't close the browser (which is already | |
| 329 // closed) at the end of the test. | |
| 330 // We use DoAction("TearDown") to tear down, because it logs it and makes | |
| 331 // it easier to check for crashes when we close the browser. | |
| 332 virtual void TearDown() OVERRIDE {} | |
| 333 | |
| 334 private: | |
| 335 // Parses the init file. | |
| 336 XmlReader init_reader_; | |
| 337 | |
| 338 // Builds the output file. | |
| 339 XmlWriter xml_writer_; | |
| 340 | |
| 341 // Time the test was started. Used to find crash dumps. | |
| 342 base::Time test_start_time_; | |
| 343 | |
| 344 // Number of times the browser has crashed during this run. | |
| 345 // Used to check for new crashes. | |
| 346 int total_crashes_; | |
| 347 | |
| 348 // Used to init the init_reader_. It must exist as long as the reader does. | |
| 349 std::string xml_init_file_; | |
| 350 | |
| 351 // If true, appends the commands to the output file as they are executed. | |
| 352 // Used for debugging when automated_ui_tests.cc crashes before it outputs | |
| 353 // results. | |
| 354 bool debug_logging_enabled_; | |
| 355 | |
| 356 // A delay in second we wait for after each action. Useful for debugging. | |
| 357 int post_action_delay_; | |
| 358 | |
| 359 DISALLOW_COPY_AND_ASSIGN(AutomatedUITest); | |
| 360 }; | |
| 361 | |
| 362 #endif // CHROME_TEST_RELIABILITY_AUTOMATED_UI_TESTS_H_ | |
| OLD | NEW |