| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 | 6 |
| 7 """Unit tests for instrumentation.TestRunner.""" | 7 """Unit tests for instrumentation.TestRunner.""" |
| 8 | 8 |
| 9 # pylint: disable=W0212 | 9 # pylint: disable=W0212 |
| 10 | 10 |
| 11 import os | 11 import os |
| 12 import sys | 12 import sys |
| 13 import unittest | 13 import unittest |
| 14 | 14 |
| 15 from pylib import constants | 15 from pylib import constants |
| 16 from pylib.base import base_test_result | 16 from pylib.base import base_test_result |
| 17 from pylib.instrumentation import test_runner | 17 from pylib.instrumentation import instrumentation_test_instance |
| 18 | 18 |
| 19 sys.path.append(os.path.join( | 19 sys.path.append(os.path.join( |
| 20 constants.DIR_SOURCE_ROOT, 'third_party', 'pymock')) | 20 constants.DIR_SOURCE_ROOT, 'third_party', 'pymock')) |
| 21 import mock # pylint: disable=F0401 | 21 import mock # pylint: disable=F0401 |
| 22 | 22 |
| 23 | 23 |
| 24 class InstrumentationTestRunnerTest(unittest.TestCase): | 24 class InstrumentationTestInstanceTest(unittest.TestCase): |
| 25 | 25 |
| 26 def setUp(self): | 26 def setUp(self): |
| 27 options = mock.Mock() | 27 options = mock.Mock() |
| 28 options.tool = '' | 28 options.tool = '' |
| 29 package = mock.Mock() | |
| 30 self.instance = test_runner.TestRunner( | |
| 31 options, '123456789abcdef0', 0, package) | |
| 32 | 29 |
| 33 def testParseAmInstrumentRawOutput_nothing(self): | 30 def testParseAmInstrumentRawOutput_nothing(self): |
| 34 code, result, statuses = ( | 31 code, result, statuses = ( |
| 35 test_runner.TestRunner._ParseAmInstrumentRawOutput([''])) | 32 instrumentation_test_instance.ParseAmInstrumentRawOutput([''])) |
| 36 self.assertEqual(None, code) | 33 self.assertEqual(None, code) |
| 37 self.assertEqual([], result) | 34 self.assertEqual([], result) |
| 38 self.assertEqual([], statuses) | 35 self.assertEqual([], statuses) |
| 39 | 36 |
| 40 def testParseAmInstrumentRawOutput_noMatchingStarts(self): | 37 def testParseAmInstrumentRawOutput_noMatchingStarts(self): |
| 41 raw_output = [ | 38 raw_output = [ |
| 42 '', | 39 '', |
| 43 'this.is.a.test.package.TestClass:.', | 40 'this.is.a.test.package.TestClass:.', |
| 44 'Test result for =.', | 41 'Test result for =.', |
| 45 'Time: 1.234', | 42 'Time: 1.234', |
| 46 '', | 43 '', |
| 47 'OK (1 test)', | 44 'OK (1 test)', |
| 48 ] | 45 ] |
| 49 | 46 |
| 50 code, result, statuses = ( | 47 code, result, statuses = ( |
| 51 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 48 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
| 52 self.assertEqual(None, code) | 49 self.assertEqual(None, code) |
| 53 self.assertEqual([], result) | 50 self.assertEqual([], result) |
| 54 self.assertEqual([], statuses) | 51 self.assertEqual([], statuses) |
| 55 | 52 |
| 56 def testParseAmInstrumentRawOutput_resultAndCode(self): | 53 def testParseAmInstrumentRawOutput_resultAndCode(self): |
| 57 raw_output = [ | 54 raw_output = [ |
| 58 'INSTRUMENTATION_RESULT: foo', | 55 'INSTRUMENTATION_RESULT: foo', |
| 59 'bar', | 56 'bar', |
| 60 'INSTRUMENTATION_CODE: -1', | 57 'INSTRUMENTATION_CODE: -1', |
| 61 ] | 58 ] |
| 62 | 59 |
| 63 code, result, _ = ( | 60 code, result, _ = ( |
| 64 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 61 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
| 65 self.assertEqual(-1, code) | 62 self.assertEqual(-1, code) |
| 66 self.assertEqual(['foo', 'bar'], result) | 63 self.assertEqual(['foo', 'bar'], result) |
| 67 | 64 |
| 68 def testParseAmInstrumentRawOutput_oneStatus(self): | 65 def testParseAmInstrumentRawOutput_oneStatus(self): |
| 69 raw_output = [ | 66 raw_output = [ |
| 70 'INSTRUMENTATION_STATUS: foo=1', | 67 'INSTRUMENTATION_STATUS: foo=1', |
| 71 'INSTRUMENTATION_STATUS: bar=hello', | 68 'INSTRUMENTATION_STATUS: bar=hello', |
| 72 'INSTRUMENTATION_STATUS: world=false', | 69 'INSTRUMENTATION_STATUS: world=false', |
| 73 'INSTRUMENTATION_STATUS: class=this.is.a.test.package.TestClass', | 70 'INSTRUMENTATION_STATUS: class=this.is.a.test.package.TestClass', |
| 74 'INSTRUMENTATION_STATUS: test=testMethod', | 71 'INSTRUMENTATION_STATUS: test=testMethod', |
| 75 'INSTRUMENTATION_STATUS_CODE: 0', | 72 'INSTRUMENTATION_STATUS_CODE: 0', |
| 76 ] | 73 ] |
| 77 | 74 |
| 78 _, _, statuses = ( | 75 _, _, statuses = ( |
| 79 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 76 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
| 80 | 77 |
| 81 expected = [ | 78 expected = [ |
| 82 (0, { | 79 (0, { |
| 83 'foo': ['1'], | 80 'foo': ['1'], |
| 84 'bar': ['hello'], | 81 'bar': ['hello'], |
| 85 'world': ['false'], | 82 'world': ['false'], |
| 86 'class': ['this.is.a.test.package.TestClass'], | 83 'class': ['this.is.a.test.package.TestClass'], |
| 87 'test': ['testMethod'], | 84 'test': ['testMethod'], |
| 88 }) | 85 }) |
| 89 ] | 86 ] |
| 90 self.assertEqual(expected, statuses) | 87 self.assertEqual(expected, statuses) |
| 91 | 88 |
| 92 def testParseAmInstrumentRawOutput_multiStatus(self): | 89 def testParseAmInstrumentRawOutput_multiStatus(self): |
| 93 raw_output = [ | 90 raw_output = [ |
| 94 'INSTRUMENTATION_STATUS: class=foo', | 91 'INSTRUMENTATION_STATUS: class=foo', |
| 95 'INSTRUMENTATION_STATUS: test=bar', | 92 'INSTRUMENTATION_STATUS: test=bar', |
| 96 'INSTRUMENTATION_STATUS_CODE: 1', | 93 'INSTRUMENTATION_STATUS_CODE: 1', |
| 97 'INSTRUMENTATION_STATUS: test_skipped=true', | 94 'INSTRUMENTATION_STATUS: test_skipped=true', |
| 98 'INSTRUMENTATION_STATUS_CODE: 0', | 95 'INSTRUMENTATION_STATUS_CODE: 0', |
| 99 'INSTRUMENTATION_STATUS: class=hello', | 96 'INSTRUMENTATION_STATUS: class=hello', |
| 100 'INSTRUMENTATION_STATUS: test=world', | 97 'INSTRUMENTATION_STATUS: test=world', |
| 101 'INSTRUMENTATION_STATUS: stack=', | 98 'INSTRUMENTATION_STATUS: stack=', |
| 102 'foo/bar.py (27)', | 99 'foo/bar.py (27)', |
| 103 'hello/world.py (42)', | 100 'hello/world.py (42)', |
| 104 'test/file.py (1)', | 101 'test/file.py (1)', |
| 105 'INSTRUMENTATION_STATUS_CODE: -1', | 102 'INSTRUMENTATION_STATUS_CODE: -1', |
| 106 ] | 103 ] |
| 107 | 104 |
| 108 _, _, statuses = ( | 105 _, _, statuses = ( |
| 109 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 106 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
| 110 | 107 |
| 111 expected = [ | 108 expected = [ |
| 112 (1, {'class': ['foo'], 'test': ['bar'],}), | 109 (1, {'class': ['foo'], 'test': ['bar'],}), |
| 113 (0, {'test_skipped': ['true']}), | 110 (0, {'test_skipped': ['true']}), |
| 114 (-1, { | 111 (-1, { |
| 115 'class': ['hello'], | 112 'class': ['hello'], |
| 116 'test': ['world'], | 113 'test': ['world'], |
| 117 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)', | 114 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)', |
| 118 'test/file.py (1)'], | 115 'test/file.py (1)'], |
| 119 }), | 116 }), |
| 120 ] | 117 ] |
| 121 self.assertEqual(expected, statuses) | 118 self.assertEqual(expected, statuses) |
| 122 | 119 |
| 123 def testParseAmInstrumentRawOutput_statusResultAndCode(self): | 120 def testParseAmInstrumentRawOutput_statusResultAndCode(self): |
| 124 raw_output = [ | 121 raw_output = [ |
| 125 'INSTRUMENTATION_STATUS: class=foo', | 122 'INSTRUMENTATION_STATUS: class=foo', |
| 126 'INSTRUMENTATION_STATUS: test=bar', | 123 'INSTRUMENTATION_STATUS: test=bar', |
| 127 'INSTRUMENTATION_STATUS_CODE: 1', | 124 'INSTRUMENTATION_STATUS_CODE: 1', |
| 128 'INSTRUMENTATION_RESULT: hello', | 125 'INSTRUMENTATION_RESULT: hello', |
| 129 'world', | 126 'world', |
| 130 '', | 127 '', |
| 131 '', | 128 '', |
| 132 'INSTRUMENTATION_CODE: 0', | 129 'INSTRUMENTATION_CODE: 0', |
| 133 ] | 130 ] |
| 134 | 131 |
| 135 code, result, statuses = ( | 132 code, result, statuses = ( |
| 136 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) | 133 instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
| 137 | 134 |
| 138 self.assertEqual(0, code) | 135 self.assertEqual(0, code) |
| 139 self.assertEqual(['hello', 'world', '', ''], result) | 136 self.assertEqual(['hello', 'world', '', ''], result) |
| 140 self.assertEqual([(1, {'class': ['foo'], 'test': ['bar']})], statuses) | 137 self.assertEqual([(1, {'class': ['foo'], 'test': ['bar']})], statuses) |
| 141 | 138 |
| 142 def testGenerateTestResult_noStatus(self): | 139 def testGenerateTestResult_noStatus(self): |
| 143 result = self.instance._GenerateTestResult( | 140 result = instrumentation_test_instance.GenerateTestResult( |
| 144 'test.package.TestClass#testMethod', [], 0, 1000) | 141 'test.package.TestClass#testMethod', [], 0, 1000) |
| 145 self.assertEqual('test.package.TestClass#testMethod', result.GetName()) | 142 self.assertEqual('test.package.TestClass#testMethod', result.GetName()) |
| 146 self.assertEqual(base_test_result.ResultType.UNKNOWN, result.GetType()) | 143 self.assertEqual(base_test_result.ResultType.UNKNOWN, result.GetType()) |
| 147 self.assertEqual('', result.GetLog()) | 144 self.assertEqual('', result.GetLog()) |
| 148 self.assertEqual(1000, result.GetDuration()) | 145 self.assertEqual(1000, result.GetDuration()) |
| 149 | 146 |
| 150 def testGenerateTestResult_testPassed(self): | 147 def testGenerateTestResult_testPassed(self): |
| 151 statuses = [ | 148 statuses = [ |
| 152 (1, { | 149 (1, { |
| 153 'class': ['test.package.TestClass'], | 150 'class': ['test.package.TestClass'], |
| 154 'test': ['testMethod'], | 151 'test': ['testMethod'], |
| 155 }), | 152 }), |
| 156 (0, { | 153 (0, { |
| 157 'class': ['test.package.TestClass'], | 154 'class': ['test.package.TestClass'], |
| 158 'test': ['testMethod'], | 155 'test': ['testMethod'], |
| 159 }), | 156 }), |
| 160 ] | 157 ] |
| 161 result = self.instance._GenerateTestResult( | 158 result = instrumentation_test_instance.GenerateTestResult( |
| 162 'test.package.TestClass#testMethod', statuses, 0, 1000) | 159 'test.package.TestClass#testMethod', statuses, 0, 1000) |
| 163 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) | 160 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) |
| 164 | 161 |
| 165 def testGenerateTestResult_testSkipped_first(self): | 162 def testGenerateTestResult_testSkipped_first(self): |
| 166 statuses = [ | 163 statuses = [ |
| 167 (0, { | 164 (0, { |
| 168 'test_skipped': ['true'], | 165 'test_skipped': ['true'], |
| 169 }), | 166 }), |
| 170 (1, { | 167 (1, { |
| 171 'class': ['test.package.TestClass'], | 168 'class': ['test.package.TestClass'], |
| 172 'test': ['testMethod'], | 169 'test': ['testMethod'], |
| 173 }), | 170 }), |
| 174 (0, { | 171 (0, { |
| 175 'class': ['test.package.TestClass'], | 172 'class': ['test.package.TestClass'], |
| 176 'test': ['testMethod'], | 173 'test': ['testMethod'], |
| 177 }), | 174 }), |
| 178 ] | 175 ] |
| 179 result = self.instance._GenerateTestResult( | 176 result = instrumentation_test_instance.GenerateTestResult( |
| 180 'test.package.TestClass#testMethod', statuses, 0, 1000) | 177 'test.package.TestClass#testMethod', statuses, 0, 1000) |
| 181 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) | 178 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) |
| 182 | 179 |
| 183 def testGenerateTestResult_testSkipped_last(self): | 180 def testGenerateTestResult_testSkipped_last(self): |
| 184 statuses = [ | 181 statuses = [ |
| 185 (1, { | 182 (1, { |
| 186 'class': ['test.package.TestClass'], | 183 'class': ['test.package.TestClass'], |
| 187 'test': ['testMethod'], | 184 'test': ['testMethod'], |
| 188 }), | 185 }), |
| 189 (0, { | 186 (0, { |
| 190 'class': ['test.package.TestClass'], | 187 'class': ['test.package.TestClass'], |
| 191 'test': ['testMethod'], | 188 'test': ['testMethod'], |
| 192 }), | 189 }), |
| 193 (0, { | 190 (0, { |
| 194 'test_skipped': ['true'], | 191 'test_skipped': ['true'], |
| 195 }), | 192 }), |
| 196 ] | 193 ] |
| 197 result = self.instance._GenerateTestResult( | 194 result = instrumentation_test_instance.GenerateTestResult( |
| 198 'test.package.TestClass#testMethod', statuses, 0, 1000) | 195 'test.package.TestClass#testMethod', statuses, 0, 1000) |
| 199 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) | 196 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) |
| 200 | 197 |
| 201 def testGenerateTestResult_testSkipped_false(self): | 198 def testGenerateTestResult_testSkipped_false(self): |
| 202 statuses = [ | 199 statuses = [ |
| 203 (0, { | 200 (0, { |
| 204 'test_skipped': ['false'], | 201 'test_skipped': ['false'], |
| 205 }), | 202 }), |
| 206 (1, { | 203 (1, { |
| 207 'class': ['test.package.TestClass'], | 204 'class': ['test.package.TestClass'], |
| 208 'test': ['testMethod'], | 205 'test': ['testMethod'], |
| 209 }), | 206 }), |
| 210 (0, { | 207 (0, { |
| 211 'class': ['test.package.TestClass'], | 208 'class': ['test.package.TestClass'], |
| 212 'test': ['testMethod'], | 209 'test': ['testMethod'], |
| 213 }), | 210 }), |
| 214 ] | 211 ] |
| 215 result = self.instance._GenerateTestResult( | 212 result = instrumentation_test_instance.GenerateTestResult( |
| 216 'test.package.TestClass#testMethod', statuses, 0, 1000) | 213 'test.package.TestClass#testMethod', statuses, 0, 1000) |
| 217 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) | 214 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) |
| 218 | 215 |
| 219 def testGenerateTestResult_testFailed(self): | 216 def testGenerateTestResult_testFailed(self): |
| 220 statuses = [ | 217 statuses = [ |
| 221 (1, { | 218 (1, { |
| 222 'class': ['test.package.TestClass'], | 219 'class': ['test.package.TestClass'], |
| 223 'test': ['testMethod'], | 220 'test': ['testMethod'], |
| 224 }), | 221 }), |
| 225 (-2, { | 222 (-2, { |
| 226 'class': ['test.package.TestClass'], | 223 'class': ['test.package.TestClass'], |
| 227 'test': ['testMethod'], | 224 'test': ['testMethod'], |
| 228 }), | 225 }), |
| 229 ] | 226 ] |
| 230 self.instance.device.old_interface.DismissCrashDialogIfNeeded = mock.Mock( | 227 result = instrumentation_test_instance.GenerateTestResult( |
| 231 return_value=None) | |
| 232 result = self.instance._GenerateTestResult( | |
| 233 'test.package.TestClass#testMethod', statuses, 0, 1000) | 228 'test.package.TestClass#testMethod', statuses, 0, 1000) |
| 234 self.assertEqual(base_test_result.ResultType.FAIL, result.GetType()) | 229 self.assertEqual(base_test_result.ResultType.FAIL, result.GetType()) |
| 235 | 230 |
| 236 def testGenerateTestResult_testCrashed(self): | |
| 237 self.instance.test_pkg.GetPackageName = mock.Mock( | |
| 238 return_value='generate.test.result.test.package') | |
| 239 self.instance.device.old_interface.DismissCrashDialogIfNeeded = mock.Mock( | |
| 240 return_value='generate.test.result.test.package') | |
| 241 statuses = [ | |
| 242 (1, { | |
| 243 'class': ['test.package.TestClass'], | |
| 244 'test': ['testMethod'], | |
| 245 }), | |
| 246 (-1, { | |
| 247 'class': ['test.package.TestClass'], | |
| 248 'test': ['testMethod'], | |
| 249 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)'], | |
| 250 }), | |
| 251 ] | |
| 252 result = self.instance._GenerateTestResult( | |
| 253 'test.package.TestClass#testMethod', statuses, 0, 1000) | |
| 254 self.assertEqual(base_test_result.ResultType.CRASH, result.GetType()) | |
| 255 self.assertEqual('\nfoo/bar.py (27)\nhello/world.py (42)', result.GetLog()) | |
| 256 | |
| 257 def test_RunTest_verifyAdbShellCommand(self): | |
| 258 self.instance.options.test_runner = 'MyTestRunner' | |
| 259 self.instance.device.StartInstrumentation = mock.Mock() | |
| 260 self.instance.test_pkg.GetPackageName = mock.Mock( | |
| 261 return_value='test.package') | |
| 262 self.instance._GetInstrumentationArgs = mock.Mock( | |
| 263 return_value={'test_arg_key': 'test_arg_value'}) | |
| 264 self.instance._RunTest('test.package.TestClass#testMethod', 100) | |
| 265 self.instance.device.StartInstrumentation.assert_called_with( | |
| 266 'test.package/MyTestRunner', raw=True, | |
| 267 extras={ | |
| 268 'test_arg_key': 'test_arg_value', | |
| 269 'class': 'test.package.TestClass#testMethod' | |
| 270 }, | |
| 271 timeout=100, retries=0) | |
| 272 | 231 |
| 273 if __name__ == '__main__': | 232 if __name__ == '__main__': |
| 274 unittest.main(verbosity=2) | 233 unittest.main(verbosity=2) |
| 275 | |
| OLD | NEW |