OLD | NEW |
1 # -*- coding: utf-8 -*- | 1 # -*- coding: utf-8 -*- |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Encapsulates running tests defined in tests.py. | 6 """Encapsulates running tests defined in tests.py. |
7 | 7 |
8 Running this script requires passing --config-path with a path to a config file | 8 Running this script requires passing --config-path with a path to a config file |
9 of the following structure: | 9 of the following structure: |
10 | 10 |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
84 | 84 |
85 | 85 |
86 def LogResultsOfTestRun(config, results): | 86 def LogResultsOfTestRun(config, results): |
87 """ Logs |results| of a test run. """ | 87 """ Logs |results| of a test run. """ |
88 logger = logging.getLogger("run_tests") | 88 logger = logging.getLogger("run_tests") |
89 failed_tests = [] | 89 failed_tests = [] |
90 failed_tests_num = 0 | 90 failed_tests_num = 0 |
91 for result in results: | 91 for result in results: |
92 website, test_case, success, reason = result | 92 website, test_case, success, reason = result |
93 if not (config.save_only_fails and success): | 93 if not (config.save_only_fails and success): |
94 logger.debug("Test case %s has %s on Website %s", test_case, | |
95 website, {True: "passed", False: "failed"}[success]) | |
96 if not success: | 94 if not success: |
97 logger.debug("Reason of failure: %s", reason) | 95 logger.debug("%s.%s failed with reason: %s", |
| 96 website, test_case, reason) |
98 | 97 |
99 if not success: | 98 if not success: |
100 failed_tests.append("%s.%s" % (website, test_case)) | 99 failed_tests.append("%s.%s" % (website, test_case)) |
101 failed_tests_num += 1 | 100 failed_tests_num += 1 |
102 | 101 |
103 logger.info("%d failed test cases out of %d, failing test cases: %s", | 102 logger.info("%d failed test cases out of %d, failing test cases: %s", |
104 failed_tests_num, len(results), | 103 failed_tests_num, len(results), |
105 sorted([name for name in failed_tests])) | 104 sorted([name for name in failed_tests])) |
106 | 105 |
107 | 106 |
108 def RunTestCaseOnWebsite((website, test_case, config)): | 107 def RunTestCaseOnWebsite((website, test_case, config)): |
109 """ Runs a |test_case| on a |website|. In case when |test_case| has | 108 """ Runs a |test_case| on a |website|. In case when |test_case| has |
110 failed it tries to rerun it. If run takes too long, then it is stopped. | 109 failed it tries to rerun it. If run takes too long, then it is stopped. |
111 """ | 110 """ |
112 | 111 |
113 profile_path = tempfile.mkdtemp() | 112 profile_path = tempfile.mkdtemp() |
114 # The tests can be flaky. This is why we try to rerun up to 3 times. | 113 # The tests can be flaky. This is why we try to rerun up to 3 times. |
115 attempts = 3 | 114 attempts = 3 |
116 result = ("", "", False, "") | 115 result = ("", "", False, "") |
117 logger = logging.getLogger("run_tests") | 116 logger = logging.getLogger("run_tests") |
118 for _ in xrange(attempts): | 117 for _ in xrange(attempts): |
119 shutil.rmtree(path=profile_path, ignore_errors=True) | 118 shutil.rmtree(path=profile_path, ignore_errors=True) |
120 logger.log(SCRIPT_DEBUG, "Run of test case %s of website %s started", | 119 logger.log(SCRIPT_DEBUG, "Run of test case %s of website %s started", |
121 test_case, website) | 120 test_case, website) |
122 try: | 121 try: |
123 with stopit.ThreadingTimeout(100) as timeout: | 122 with stopit.ThreadingTimeout(seconds=100) as timeout: |
124 logger.log(SCRIPT_DEBUG, | 123 logger.log(SCRIPT_DEBUG, |
125 "Run test with parameters: %s %s %s %s %s %s", | 124 "Run test with parameters: %s %s %s %s %s %s", |
126 config.chrome_path, config.chromedriver_path, | 125 config.chrome_path, config.chromedriver_path, |
127 profile_path, config.passwords_path, | 126 profile_path, config.passwords_path, |
128 website, test_case) | 127 website, test_case) |
129 result = tests.RunTest(config.chrome_path, config.chromedriver_path, | 128 result = tests.RunTest(config.chrome_path, config.chromedriver_path, |
130 profile_path, config.passwords_path, | 129 profile_path, config.passwords_path, |
131 website, test_case)[0] | 130 website, test_case)[0] |
132 if timeout.state != timeout.EXECUTED: | 131 if timeout.state != timeout.EXECUTED: |
133 result = (website, test_case, False, | 132 result = (website, test_case, False, |
134 "Got %d as timeout state (see stopit.ThreadingTimeout for" | 133 "Got %d as timeout state (see stopit.ThreadingTimeout for" |
135 "the meaning of the number)" % timeout.state) | 134 " the meaning of the number)" % timeout.state) |
136 _, _, success, _ = result | 135 _, _, success, _ = result |
137 if success: | 136 if success: |
138 return result | 137 return result |
139 except Exception as e: | 138 except Exception as e: |
140 result = (website, test_case, False, e) | 139 result = (website, test_case, False, e) |
141 return result | 140 return result |
142 | 141 |
143 | 142 |
144 def RunTests(config_path): | 143 def RunTests(config_path): |
145 """Runs automated tests. | 144 """Runs automated tests. |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
177 def main(): | 176 def main(): |
178 parser = argparse.ArgumentParser() | 177 parser = argparse.ArgumentParser() |
179 parser.add_argument("config_path", metavar="N", | 178 parser.add_argument("config_path", metavar="N", |
180 help="Path to the config.ini file.") | 179 help="Path to the config.ini file.") |
181 args = parser.parse_args() | 180 args = parser.parse_args() |
182 RunTests(args.config_path) | 181 RunTests(args.config_path) |
183 | 182 |
184 | 183 |
185 if __name__ == "__main__": | 184 if __name__ == "__main__": |
186 main() | 185 main() |
OLD | NEW |