OLD | NEW |
| (Empty) |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #ifndef CHROME_TEST_RELIABILITY_AUTOMATED_UI_TESTS_H_ | |
6 #define CHROME_TEST_RELIABILITY_AUTOMATED_UI_TESTS_H_ | |
7 | |
8 // This takes an input file of commands, which consist of a series of | |
9 // actions, and runs every command, reporting the status of each one | |
10 // to an output file once all the commands have been run. | |
11 // | |
12 // The input file should be an XML file that has a root of any name followed | |
13 // by a series of elements named "command" with child elements representing | |
14 // the various actions, in order, to be performed during each command. A | |
15 // command element can optionally include an "number" attribute to identify it. | |
16 // | |
17 // Example: | |
18 // <CommandList> | |
19 // <command number="1"><NewTab/><Navigate/><OpenWindow/><Navigate/><Back/> | |
20 // </command> | |
21 // <command number="2"><NewTab/><Navigate/><Navigate/><Back/><Forward/> | |
22 // </command> | |
23 // <command number="3"><CloseTab/><OpenWindow/><NewTab/><Navigate/><CloseTab/> | |
24 // </command> | |
25 // </CommandList> | |
26 // | |
27 // When the test is finished it will output results to the output file, | |
28 // overwriting any previous version of this file. The output file is an | |
29 // XML file which reports on each command, indicating whether it successfully | |
30 // ran and if there were any errors. | |
31 // | |
32 // Example: (actual output will probably contain more actions per command): | |
33 // <Report> | |
34 // <Executed command_number="1"><NewTab/><Navigate/><result><success/> | |
35 // </result> </Executed> | |
36 // <Executed command_number="2"><Back/><Forward/><result><success/></result> | |
37 // </Executed> | |
38 // <Executed command_number="3"><CloseTab/><result> | |
39 // <crash crash_dump="C:\a_crash.txt" command_completed="no"/></result> | |
40 // </Executed> | |
41 // </Report> | |
42 // | |
43 // A "crash" result will have two attributes, crash_dump, which points | |
44 // to the full path of crash dump associated with this crash, and | |
45 // command_completed which indicates whether or not the last | |
46 // action recorded was the final action of the command. | |
47 // | |
48 // Furthermore, each individual action may contain additional attributes | |
49 // to log non-fatal failures. If the attribute 'failed_to_complete="yes"' | |
50 // is present, then the action did not complete. If that attribute is present, | |
51 // an info, warning, or error attribute will also be present, and will contain | |
52 // a string describing the error. The presence of info means the failure was | |
53 // expected, probably due to a state making the action impossible to perform | |
54 // like trying to close the last remaining window. Warnings usually mean the | |
55 // action couldn't complete for an unknown and unexpected reason, but that the | |
56 // test state is probably fine. Errors are like warnings, but they mean the test | |
57 // state is probably incorrect, and more failures are likely to be caused by | |
58 // the same problem. | |
59 // | |
60 // Example of some failed actions: | |
61 // <CloseTab failed_to_complete="yes" info="would_have_exited_application"/> | |
62 // <Reload failed_to_complete="yes" warning="failed_to_apply_accelerator"/> | |
63 // <Star failed_to_complete="yes" error="browser_window_not_found"/> | |
64 // | |
65 // | |
66 // Switches | |
67 // --input : Specifies the input file, must be an absolute directory. | |
68 // Default is "C:\automated_ui_tests.txt" | |
69 // | |
70 // --output : Specifies the output file, must be an absolute directory. | |
71 // Default is "C:\automated_ui_tests_error_report.txt" | |
72 // | |
73 // | |
74 // Test reproduction options: | |
75 // | |
76 // If you're trying to reproduce the results from crash reports use the | |
77 // following switches | |
78 // | |
79 // --key : Specifies, via a comma delimited list, what actions to run. Examples: | |
80 // --key=SetUp,ZoomPlus,Forward,History,Navigate,Back,TearDown | |
81 // --key=SetUp,ZoomPlus | |
82 // Note, the second key doesn't include a TearDown, that will | |
83 // automatically be added if the result doesn't crash. | |
84 // | |
85 // --num-reproductions : Specifies the number of reproductions to run, the | |
86 // default is 1. Suggested use: run without this flag | |
87 // to see if we reproduce a crash, then run with the flag | |
88 // if there isn't a crash, to see if it might be a rare | |
89 // race condition that causes the crash. | |
90 // | |
91 // | |
92 // Debugging options: | |
93 // | |
94 // --debug : Will append each action that is performed to the output file, as | |
95 // soon as the action is performed. If the program finishes, this file | |
96 // will be overwritten with the normal results. This flag is used to | |
97 // help debug the tests if they are crashing before they get a chance | |
98 // to write their results to file. | |
99 // | |
100 // --wait-after-action : waits the specified amount of time (1s by default) | |
101 // after each action. Useful for debugging. | |
102 | |
103 #include <string> | |
104 | |
105 #include "chrome/test/reliability/automated_ui_test_base.h" | |
106 #include "chrome/test/ui/ui_test.h" | |
107 #include "third_party/libxml/chromium/libxml_utils.h" | |
108 #include "ui/events/keycodes/keyboard_codes.h" | |
109 | |
110 namespace base { | |
111 class Time; | |
112 } | |
113 | |
114 class AutomatedUITest : public AutomatedUITestBase { | |
115 protected: | |
116 AutomatedUITest(); | |
117 virtual ~AutomatedUITest(); | |
118 | |
119 // Runs a reproduction of one set of actions, reporting whether they crash | |
120 // or not. | |
121 void RunReproduction(); | |
122 | |
123 // Runs automated UI tests which are read from the input file. | |
124 // Reports crashes to the output file. | |
125 void RunAutomatedUITest(); | |
126 | |
127 // Attempts to perform an action based on the input string. See below for | |
128 // possible actions. Returns true if the action completes, false otherwise. | |
129 bool DoAction(const std::string& action); | |
130 | |
131 // Actions ------------------------------------------------------------------ | |
132 | |
133 // NOTE: This list is sorted alphabetically, so that we can easily detect | |
134 // missing actions. | |
135 | |
136 // Changes the encoding of the page (the encoding is selected at random | |
137 // from a list of encodings). | |
138 // Returns true if call to activate the accelerator is successful. | |
139 // XML element: <ChangeEncoding/> | |
140 bool ChangeEncoding(); | |
141 | |
142 // Opens one of the dialogs (chosen randomly) and exercises it. | |
143 // XML element: <Dialog/> | |
144 bool ExerciseDialog(); | |
145 | |
146 // Opens the JavaScriptConsole window. While it isn't modal, it takes focus | |
147 // from the current browser window, so most of the test can't continue until | |
148 // it is dismissed. | |
149 // XML element: <JavaScriptConsole/> | |
150 bool JavaScriptConsole(); | |
151 | |
152 // Opens the About dialog. This dialog is modal so a majority of the test | |
153 // can't be completed until it is dismissed. | |
154 // XML element: <About/> | |
155 bool OpenAboutDialog(); | |
156 | |
157 // Opens the Clear Browsing Data dialog, this dialog is modal so a majority of | |
158 // the test can't be completed until it is dismissed. | |
159 // XML element: <ClearBrowsingData/> | |
160 bool OpenClearBrowsingDataDialog(); | |
161 | |
162 // Opens the Search Engines dialog. While it isn't modal, it takes focus from | |
163 // the current browser window, so most of the test can't continue until it is | |
164 // dismissed. | |
165 // XML element: <EditSearchEngines/> | |
166 bool OpenEditSearchEnginesDialog(); | |
167 | |
168 // Opens the Import Settings dialog, this dialog is modal so a majority of | |
169 // the test can't be completed until it is dismissed. | |
170 // XML element: <ImportSettings/> | |
171 bool OpenImportSettingsDialog(); | |
172 | |
173 // Opens the Task Manager dialog. While it isn't modal, it takes focus from | |
174 // the current browser window, so most of the test can't continue until it is | |
175 // dismissed. | |
176 // XML element: <TaskManager/> | |
177 bool OpenTaskManagerDialog(); | |
178 | |
179 // Opens the View Passwords dialog, this dialog is modal so a majority of | |
180 // the test can't be completed until it is dismissed. | |
181 // XML element: <ViewPasswords/> | |
182 bool OpenViewPasswordsDialog(); | |
183 | |
184 // Opens the Options dialog. While it isn't modal, it takes focus from | |
185 // the current browser window, so most of the test can't continue until it is | |
186 // dismissed. | |
187 // XML element: <Options/> | |
188 bool Options(); | |
189 | |
190 // Stars the current page. This opens a dialog that may or may not be | |
191 // dismissed. | |
192 // XML element: <Star/> | |
193 bool StarPage(); | |
194 | |
195 // Views source of the current page. | |
196 // Returns true if call to activate the accelerator is successful. | |
197 // XML element: <ViewSource/> | |
198 bool ViewSource(); | |
199 | |
200 // Decreases the text size on the current active tab. | |
201 // XML element: <ZoomMinus/> | |
202 bool ZoomMinus(); | |
203 | |
204 // Increases the text size on the current active tab. | |
205 // XML element: <ZoomPlus/> | |
206 bool ZoomPlus(); | |
207 | |
208 // Test Dialog Actions ****************************************************** | |
209 // These are a special set of actions that perform multiple actions on a | |
210 // specified dialog. They run kTestDialogActionsToRun actions randomly | |
211 // chosen from test_dialog_possible_actions_ after opening the dialog. They | |
212 // then always end with a PressEscapeKey action, to attempt to close the | |
213 // dialog. | |
214 // | |
215 // The randomly performed actions are logged as child elements of the | |
216 // TestDialog action. For example (for kTestDialogActionsToRun = 4): | |
217 // <TestEditKeywords> <PressTabKey/><PressEnterKey/><DownArrow/> | |
218 // <DownArrow/><PressEscapeKey/> </TestEditKeywords> | |
219 | |
220 // Opens About dialog and runs random actions on it. | |
221 // XML element: <TestAboutChrome/> | |
222 bool TestAboutChrome(); | |
223 | |
224 // Opens Clear Browsing Data dialog and runs random actions on it. | |
225 // XML element: <TestClearBrowsingData/> | |
226 bool TestClearBrowsingData(); | |
227 | |
228 // Opens Edit Keywords dialog and runs random actions on it. | |
229 // XML element: <TestEditSearchEngines/> | |
230 bool TestEditSearchEngines(); | |
231 | |
232 // Opens Import Settings dialog and runs random actions on it. | |
233 // XML element: <TestImportSettings/> | |
234 bool TestImportSettings(); | |
235 | |
236 // Opens Options dialog and runs random actions on it. | |
237 // XML element: <TestOptions/> | |
238 bool TestOptions(); | |
239 | |
240 // Opens Task Manager and runs random actions on it. | |
241 // This has the possibility of killing both the browser and renderer | |
242 // processes, which will cause non-fatal errors for the remaining actions | |
243 // in this command. | |
244 // XML element: <TestTaskManager/> | |
245 bool TestTaskManager(); | |
246 | |
247 // Opens View Passwords dialog and runs random actions on it. | |
248 // XML element: <TestViewPasswords/> | |
249 bool TestViewPasswords(); | |
250 | |
251 // End Test Dialog Actions ************************************************** | |
252 | |
253 // Runs a limited set of actions designed to test dialogs. Will run | |
254 // |num_actions| from the set defined in test_dialog_possible_actions_. | |
255 bool FuzzyTestDialog(int num_actions); | |
256 | |
257 // Navigates to about:crash. | |
258 // XML element: <Crash/> | |
259 bool ForceCrash(); | |
260 | |
261 // Utility functions -------------------------------------------------------- | |
262 | |
263 // Opens init file, reads it into the reader, and closes the file. | |
264 // Returns false if there are any errors. | |
265 bool InitXMLReader(); | |
266 | |
267 // Closes the xml_writer and outputs the contents of its buffer to | |
268 // the output file. | |
269 bool WriteReportToFile(); | |
270 | |
271 // Appends the provided string to the output file. | |
272 void AppendToOutputFile(const std::string& append_string); | |
273 | |
274 // Logs a crash to the xml_writer in the form of: | |
275 // <result><crash crash_dump="|crash_dump|" command_completed="yes/no"/> | |
276 // </result> | |
277 // crash_dump - Location of crash dump if applicable. | |
278 // command_completed - True if all actions in the command were completed | |
279 // before the crash occured. | |
280 void LogCrashResult(const base::FilePath& crash_dump, | |
281 bool command_completed); | |
282 | |
283 // Logs a successful command to the xml_writer in the form of: | |
284 // <result><success/><result/> | |
285 void LogSuccessResult(); | |
286 | |
287 // Adds the attribute "reason=|reason|" to the current element. | |
288 // Used to log the reason for a given failure while performing an action. | |
289 void LogActionFailureReason(const std::string& reason); | |
290 | |
291 // Adds the attribute 'info="|info|"' to the current element. Used when an | |
292 // action could not complete for a non-serious issue. Usually because the | |
293 // state of the test wouldn't allow for a particular action. | |
294 void AddInfoAttribute(const std::string& info); | |
295 | |
296 // Adds the attribute "warning=|warning|" to the current element. Used when | |
297 // an action could not complete because of a potentially troublesome issue. | |
298 void AddWarningAttribute(const std::string& warning); | |
299 | |
300 // Adds the attribute "error=|error|" to the current element. Used when an | |
301 // action could not complete due to an unexpected problem which might | |
302 // invalidate the results of the entire command (not just the action). | |
303 // This is usually used when the testing environment isn't acting as we'd | |
304 // expect. For example, no chrome windows are focused, or key presses aren't | |
305 // being registered. | |
306 void AddErrorAttribute(const std::string& error); | |
307 | |
308 // Returns the full path of the crash dump. This is likely to be the | |
309 // .txt file, not the actual crash dump. Although they do share | |
310 // a common name. | |
311 base::FilePath GetMostRecentCrashDump(); | |
312 | |
313 // Returns true if the test has produced any new crash logs. | |
314 // A "new" crash log is one that was produced since DidCrash was last called | |
315 // with |update_total_crashes| set to true. | |
316 bool DidCrash(bool update_total_crashes); | |
317 | |
318 // Override the message logging in AutomatedUITestBase. | |
319 virtual void LogErrorMessage(const std::string& error) OVERRIDE; | |
320 virtual void LogWarningMessage(const std::string& warning) OVERRIDE; | |
321 virtual void LogInfoMessage(const std::string& info) OVERRIDE; | |
322 | |
323 // Overridden so that UI Test doesn't set up when the tests start. | |
324 // We use DoAction("SetUp") to set up, because it logs it and makes | |
325 // it easier to check for crashes when we start the browser. | |
326 virtual void SetUp() OVERRIDE {} | |
327 | |
328 // Overridden so that UI Test doesn't close the browser (which is already | |
329 // closed) at the end of the test. | |
330 // We use DoAction("TearDown") to tear down, because it logs it and makes | |
331 // it easier to check for crashes when we close the browser. | |
332 virtual void TearDown() OVERRIDE {} | |
333 | |
334 private: | |
335 // Parses the init file. | |
336 XmlReader init_reader_; | |
337 | |
338 // Builds the output file. | |
339 XmlWriter xml_writer_; | |
340 | |
341 // Time the test was started. Used to find crash dumps. | |
342 base::Time test_start_time_; | |
343 | |
344 // Number of times the browser has crashed during this run. | |
345 // Used to check for new crashes. | |
346 int total_crashes_; | |
347 | |
348 // Used to init the init_reader_. It must exist as long as the reader does. | |
349 std::string xml_init_file_; | |
350 | |
351 // If true, appends the commands to the output file as they are executed. | |
352 // Used for debugging when automated_ui_tests.cc crashes before it outputs | |
353 // results. | |
354 bool debug_logging_enabled_; | |
355 | |
356 // A delay in second we wait for after each action. Useful for debugging. | |
357 int post_action_delay_; | |
358 | |
359 DISALLOW_COPY_AND_ASSIGN(AutomatedUITest); | |
360 }; | |
361 | |
362 #endif // CHROME_TEST_RELIABILITY_AUTOMATED_UI_TESTS_H_ | |
OLD | NEW |