OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 | 6 |
7 """Unit tests for instrumentation.TestRunner.""" | 7 """Unit tests for instrumentation.TestRunner.""" |
8 | 8 |
9 # pylint: disable=W0212 | 9 # pylint: disable=W0212 |
10 | 10 |
11 import os | 11 import os |
12 import sys | 12 import sys |
13 import unittest | 13 import unittest |
14 | 14 |
15 from pylib import constants | 15 from pylib import constants |
16 from pylib.base import base_test_result | 16 from pylib.base import base_test_result |
17 from pylib.instrumentation import test_runner | 17 from pylib.instrumentation import instrumentation_test_instance |
18 | 18 |
19 sys.path.append(os.path.join( | 19 sys.path.append(os.path.join( |
20 constants.DIR_SOURCE_ROOT, 'third_party', 'pymock')) | 20 constants.DIR_SOURCE_ROOT, 'third_party', 'pymock')) |
21 import mock # pylint: disable=F0401 | 21 import mock # pylint: disable=F0401 |
22 | 22 |
23 | 23 |
24 class InstrumentationTestRunnerTest(unittest.TestCase): | 24 class InstrumentationTestInstanceTest(unittest.TestCase): |
25 | 25 |
26 def setUp(self): | 26 def setUp(self): |
27 options = mock.Mock() | 27 options = mock.Mock() |
28 options.tool = '' | 28 options.tool = '' |
29 package = mock.Mock() | |
30 self.instance = test_runner.TestRunner( | |
31 options, '123456789abcdef0', 0, package) | |
32 | 29 |
33 def testParseAmInstrumentRawOutput_nothing(self): | 30 def testParseAmInstrumentRawOutput_nothing(self): |
34 code, result, statuses = ( | 31 code, result, statuses = ( |
35 test_runner.TestRunner._ParseAmInstrumentRawOutput([''])) | 32 instrumentation_test_instance.ParseAmInstrumentRawOutput([''])) |
36 self.assertEqual(None, code) | 33 self.assertEqual(None, code) |
37 self.assertEqual([], result) | 34 self.assertEqual([], result) |
38 self.assertEqual([], statuses) | 35 self.assertEqual([], statuses) |
39 | 36 |
40 def testParseAmInstrumentRawOutput_noMatchingStarts(self): | 37 def testParseAmInstrumentRawOutput_noMatchingStarts(self): |
41 raw_output = [ | 38 raw_output = [ |
42 '', | 39 '', |
43 'this.is.a.test.package.TestClass:.', | 40 'this.is.a.test.package.TestClass:.', |
44 'Test result for =.', | 41 'Test result for =.', |
45 'Time: 1.234', | 42 'Time: 1.234', |
46 '', | 43 '', |
47 'OK (1 test)', | 44 'OK (1 test)', |
48 ] | 45 ] |
49 | 46 |
50 code, result, statuses = ( | 47 code, result, statuses = ( |
51 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 48 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
52 self.assertEqual(None, code) | 49 self.assertEqual(None, code) |
53 self.assertEqual([], result) | 50 self.assertEqual([], result) |
54 self.assertEqual([], statuses) | 51 self.assertEqual([], statuses) |
55 | 52 |
56 def testParseAmInstrumentRawOutput_resultAndCode(self): | 53 def testParseAmInstrumentRawOutput_resultAndCode(self): |
57 raw_output = [ | 54 raw_output = [ |
58 'INSTRUMENTATION_RESULT: foo', | 55 'INSTRUMENTATION_RESULT: foo', |
59 'bar', | 56 'bar', |
60 'INSTRUMENTATION_CODE: -1', | 57 'INSTRUMENTATION_CODE: -1', |
61 ] | 58 ] |
62 | 59 |
63 code, result, _ = ( | 60 code, result, _ = ( |
64 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 61 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
65 self.assertEqual(-1, code) | 62 self.assertEqual(-1, code) |
66 self.assertEqual(['foo', 'bar'], result) | 63 self.assertEqual(['foo', 'bar'], result) |
67 | 64 |
68 def testParseAmInstrumentRawOutput_oneStatus(self): | 65 def testParseAmInstrumentRawOutput_oneStatus(self): |
69 raw_output = [ | 66 raw_output = [ |
70 'INSTRUMENTATION_STATUS: foo=1', | 67 'INSTRUMENTATION_STATUS: foo=1', |
71 'INSTRUMENTATION_STATUS: bar=hello', | 68 'INSTRUMENTATION_STATUS: bar=hello', |
72 'INSTRUMENTATION_STATUS: world=false', | 69 'INSTRUMENTATION_STATUS: world=false', |
73 'INSTRUMENTATION_STATUS: class=this.is.a.test.package.TestClass', | 70 'INSTRUMENTATION_STATUS: class=this.is.a.test.package.TestClass', |
74 'INSTRUMENTATION_STATUS: test=testMethod', | 71 'INSTRUMENTATION_STATUS: test=testMethod', |
75 'INSTRUMENTATION_STATUS_CODE: 0', | 72 'INSTRUMENTATION_STATUS_CODE: 0', |
76 ] | 73 ] |
77 | 74 |
78 _, _, statuses = ( | 75 _, _, statuses = ( |
79 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 76 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
80 | 77 |
81 expected = [ | 78 expected = [ |
82 (0, { | 79 (0, { |
83 'foo': ['1'], | 80 'foo': ['1'], |
84 'bar': ['hello'], | 81 'bar': ['hello'], |
85 'world': ['false'], | 82 'world': ['false'], |
86 'class': ['this.is.a.test.package.TestClass'], | 83 'class': ['this.is.a.test.package.TestClass'], |
87 'test': ['testMethod'], | 84 'test': ['testMethod'], |
88 }) | 85 }) |
89 ] | 86 ] |
90 self.assertEqual(expected, statuses) | 87 self.assertEqual(expected, statuses) |
91 | 88 |
92 def testParseAmInstrumentRawOutput_multiStatus(self): | 89 def testParseAmInstrumentRawOutput_multiStatus(self): |
93 raw_output = [ | 90 raw_output = [ |
94 'INSTRUMENTATION_STATUS: class=foo', | 91 'INSTRUMENTATION_STATUS: class=foo', |
95 'INSTRUMENTATION_STATUS: test=bar', | 92 'INSTRUMENTATION_STATUS: test=bar', |
96 'INSTRUMENTATION_STATUS_CODE: 1', | 93 'INSTRUMENTATION_STATUS_CODE: 1', |
97 'INSTRUMENTATION_STATUS: test_skipped=true', | 94 'INSTRUMENTATION_STATUS: test_skipped=true', |
98 'INSTRUMENTATION_STATUS_CODE: 0', | 95 'INSTRUMENTATION_STATUS_CODE: 0', |
99 'INSTRUMENTATION_STATUS: class=hello', | 96 'INSTRUMENTATION_STATUS: class=hello', |
100 'INSTRUMENTATION_STATUS: test=world', | 97 'INSTRUMENTATION_STATUS: test=world', |
101 'INSTRUMENTATION_STATUS: stack=', | 98 'INSTRUMENTATION_STATUS: stack=', |
102 'foo/bar.py (27)', | 99 'foo/bar.py (27)', |
103 'hello/world.py (42)', | 100 'hello/world.py (42)', |
104 'test/file.py (1)', | 101 'test/file.py (1)', |
105 'INSTRUMENTATION_STATUS_CODE: -1', | 102 'INSTRUMENTATION_STATUS_CODE: -1', |
106 ] | 103 ] |
107 | 104 |
108 _, _, statuses = ( | 105 _, _, statuses = ( |
109 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 106 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
110 | 107 |
111 expected = [ | 108 expected = [ |
112 (1, {'class': ['foo'], 'test': ['bar'],}), | 109 (1, {'class': ['foo'], 'test': ['bar'],}), |
113 (0, {'test_skipped': ['true']}), | 110 (0, {'test_skipped': ['true']}), |
114 (-1, { | 111 (-1, { |
115 'class': ['hello'], | 112 'class': ['hello'], |
116 'test': ['world'], | 113 'test': ['world'], |
117 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)', | 114 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)', |
118 'test/file.py (1)'], | 115 'test/file.py (1)'], |
119 }), | 116 }), |
120 ] | 117 ] |
121 self.assertEqual(expected, statuses) | 118 self.assertEqual(expected, statuses) |
122 | 119 |
123 def testParseAmInstrumentRawOutput_statusResultAndCode(self): | 120 def testParseAmInstrumentRawOutput_statusResultAndCode(self): |
124 raw_output = [ | 121 raw_output = [ |
125 'INSTRUMENTATION_STATUS: class=foo', | 122 'INSTRUMENTATION_STATUS: class=foo', |
126 'INSTRUMENTATION_STATUS: test=bar', | 123 'INSTRUMENTATION_STATUS: test=bar', |
127 'INSTRUMENTATION_STATUS_CODE: 1', | 124 'INSTRUMENTATION_STATUS_CODE: 1', |
128 'INSTRUMENTATION_RESULT: hello', | 125 'INSTRUMENTATION_RESULT: hello', |
129 'world', | 126 'world', |
130 '', | 127 '', |
131 '', | 128 '', |
132 'INSTRUMENTATION_CODE: 0', | 129 'INSTRUMENTATION_CODE: 0', |
133 ] | 130 ] |
134 | 131 |
135 code, result, statuses = ( | 132 code, result, statuses = ( |
136 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 133 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
137 | 134 |
138 self.assertEqual(0, code) | 135 self.assertEqual(0, code) |
139 self.assertEqual(['hello', 'world', '', ''], result) | 136 self.assertEqual(['hello', 'world', '', ''], result) |
140 self.assertEqual([(1, {'class': ['foo'], 'test': ['bar']})], statuses) | 137 self.assertEqual([(1, {'class': ['foo'], 'test': ['bar']})], statuses) |
141 | 138 |
142 def testGenerateTestResult_noStatus(self): | 139 def testGenerateTestResult_noStatus(self): |
143 result = self.instance._GenerateTestResult( | 140 result = instrumentation_test_instance.GenerateTestResult( |
144 'test.package.TestClass#testMethod', [], 0, 1000) | 141 'test.package.TestClass#testMethod', [], 0, 1000) |
145 self.assertEqual('test.package.TestClass#testMethod', result.GetName()) | 142 self.assertEqual('test.package.TestClass#testMethod', result.GetName()) |
146 self.assertEqual(base_test_result.ResultType.UNKNOWN, result.GetType()) | 143 self.assertEqual(base_test_result.ResultType.UNKNOWN, result.GetType()) |
147 self.assertEqual('', result.GetLog()) | 144 self.assertEqual('', result.GetLog()) |
148 self.assertEqual(1000, result.GetDuration()) | 145 self.assertEqual(1000, result.GetDuration()) |
149 | 146 |
150 def testGenerateTestResult_testPassed(self): | 147 def testGenerateTestResult_testPassed(self): |
151 statuses = [ | 148 statuses = [ |
152 (1, { | 149 (1, { |
153 'class': ['test.package.TestClass'], | 150 'class': ['test.package.TestClass'], |
154 'test': ['testMethod'], | 151 'test': ['testMethod'], |
155 }), | 152 }), |
156 (0, { | 153 (0, { |
157 'class': ['test.package.TestClass'], | 154 'class': ['test.package.TestClass'], |
158 'test': ['testMethod'], | 155 'test': ['testMethod'], |
159 }), | 156 }), |
160 ] | 157 ] |
161 result = self.instance._GenerateTestResult( | 158 result = instrumentation_test_instance.GenerateTestResult( |
162 'test.package.TestClass#testMethod', statuses, 0, 1000) | 159 'test.package.TestClass#testMethod', statuses, 0, 1000) |
163 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) | 160 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) |
164 | 161 |
165 def testGenerateTestResult_testSkipped_first(self): | 162 def testGenerateTestResult_testSkipped_first(self): |
166 statuses = [ | 163 statuses = [ |
167 (0, { | 164 (0, { |
168 'test_skipped': ['true'], | 165 'test_skipped': ['true'], |
169 }), | 166 }), |
170 (1, { | 167 (1, { |
171 'class': ['test.package.TestClass'], | 168 'class': ['test.package.TestClass'], |
172 'test': ['testMethod'], | 169 'test': ['testMethod'], |
173 }), | 170 }), |
174 (0, { | 171 (0, { |
175 'class': ['test.package.TestClass'], | 172 'class': ['test.package.TestClass'], |
176 'test': ['testMethod'], | 173 'test': ['testMethod'], |
177 }), | 174 }), |
178 ] | 175 ] |
179 result = self.instance._GenerateTestResult( | 176 result = instrumentation_test_instance.GenerateTestResult( |
180 'test.package.TestClass#testMethod', statuses, 0, 1000) | 177 'test.package.TestClass#testMethod', statuses, 0, 1000) |
181 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) | 178 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) |
182 | 179 |
183 def testGenerateTestResult_testSkipped_last(self): | 180 def testGenerateTestResult_testSkipped_last(self): |
184 statuses = [ | 181 statuses = [ |
185 (1, { | 182 (1, { |
186 'class': ['test.package.TestClass'], | 183 'class': ['test.package.TestClass'], |
187 'test': ['testMethod'], | 184 'test': ['testMethod'], |
188 }), | 185 }), |
189 (0, { | 186 (0, { |
190 'class': ['test.package.TestClass'], | 187 'class': ['test.package.TestClass'], |
191 'test': ['testMethod'], | 188 'test': ['testMethod'], |
192 }), | 189 }), |
193 (0, { | 190 (0, { |
194 'test_skipped': ['true'], | 191 'test_skipped': ['true'], |
195 }), | 192 }), |
196 ] | 193 ] |
197 result = self.instance._GenerateTestResult( | 194 result = instrumentation_test_instance.GenerateTestResult( |
198 'test.package.TestClass#testMethod', statuses, 0, 1000) | 195 'test.package.TestClass#testMethod', statuses, 0, 1000) |
199 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) | 196 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) |
200 | 197 |
201 def testGenerateTestResult_testSkipped_false(self): | 198 def testGenerateTestResult_testSkipped_false(self): |
202 statuses = [ | 199 statuses = [ |
203 (0, { | 200 (0, { |
204 'test_skipped': ['false'], | 201 'test_skipped': ['false'], |
205 }), | 202 }), |
206 (1, { | 203 (1, { |
207 'class': ['test.package.TestClass'], | 204 'class': ['test.package.TestClass'], |
208 'test': ['testMethod'], | 205 'test': ['testMethod'], |
209 }), | 206 }), |
210 (0, { | 207 (0, { |
211 'class': ['test.package.TestClass'], | 208 'class': ['test.package.TestClass'], |
212 'test': ['testMethod'], | 209 'test': ['testMethod'], |
213 }), | 210 }), |
214 ] | 211 ] |
215 result = self.instance._GenerateTestResult( | 212 result = instrumentation_test_instance.GenerateTestResult( |
216 'test.package.TestClass#testMethod', statuses, 0, 1000) | 213 'test.package.TestClass#testMethod', statuses, 0, 1000) |
217 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) | 214 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) |
218 | 215 |
219 def testGenerateTestResult_testFailed(self): | 216 def testGenerateTestResult_testFailed(self): |
220 statuses = [ | 217 statuses = [ |
221 (1, { | 218 (1, { |
222 'class': ['test.package.TestClass'], | 219 'class': ['test.package.TestClass'], |
223 'test': ['testMethod'], | 220 'test': ['testMethod'], |
224 }), | 221 }), |
225 (-2, { | 222 (-2, { |
226 'class': ['test.package.TestClass'], | 223 'class': ['test.package.TestClass'], |
227 'test': ['testMethod'], | 224 'test': ['testMethod'], |
228 }), | 225 }), |
229 ] | 226 ] |
230 self.instance.device.old_interface.DismissCrashDialogIfNeeded = mock.Mock( | 227 result = instrumentation_test_instance.GenerateTestResult( |
231 return_value=None) | |
232 result = self.instance._GenerateTestResult( | |
233 'test.package.TestClass#testMethod', statuses, 0, 1000) | 228 'test.package.TestClass#testMethod', statuses, 0, 1000) |
234 self.assertEqual(base_test_result.ResultType.FAIL, result.GetType()) | 229 self.assertEqual(base_test_result.ResultType.FAIL, result.GetType()) |
235 | 230 |
236 def testGenerateTestResult_testCrashed(self): | |
237 self.instance.test_pkg.GetPackageName = mock.Mock( | |
238 return_value='generate.test.result.test.package') | |
239 self.instance.device.old_interface.DismissCrashDialogIfNeeded = mock.Mock( | |
240 return_value='generate.test.result.test.package') | |
241 statuses = [ | |
242 (1, { | |
243 'class': ['test.package.TestClass'], | |
244 'test': ['testMethod'], | |
245 }), | |
246 (-1, { | |
247 'class': ['test.package.TestClass'], | |
248 'test': ['testMethod'], | |
249 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)'], | |
250 }), | |
251 ] | |
252 result = self.instance._GenerateTestResult( | |
253 'test.package.TestClass#testMethod', statuses, 0, 1000) | |
254 self.assertEqual(base_test_result.ResultType.CRASH, result.GetType()) | |
255 self.assertEqual('\nfoo/bar.py (27)\nhello/world.py (42)', result.GetLog()) | |
256 | |
257 def test_RunTest_verifyAdbShellCommand(self): | |
258 self.instance.options.test_runner = 'MyTestRunner' | |
259 self.instance.device.StartInstrumentation = mock.Mock() | |
260 self.instance.test_pkg.GetPackageName = mock.Mock( | |
261 return_value='test.package') | |
262 self.instance._GetInstrumentationArgs = mock.Mock( | |
263 return_value={'test_arg_key': 'test_arg_value'}) | |
264 self.instance._RunTest('test.package.TestClass#testMethod', 100) | |
265 self.instance.device.StartInstrumentation.assert_called_with( | |
266 'test.package/MyTestRunner', raw=True, | |
267 extras={ | |
268 'test_arg_key': 'test_arg_value', | |
269 'class': 'test.package.TestClass#testMethod' | |
270 }, | |
271 timeout=100, retries=0) | |
272 | 231 |
273 if __name__ == '__main__': | 232 if __name__ == '__main__': |
274 unittest.main(verbosity=2) | 233 unittest.main(verbosity=2) |
275 | |
OLD | NEW |