OLD | NEW |
1 # Copyright (c) 2012 Google Inc. All rights reserved. | 1 # Copyright (c) 2012 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 23 matching lines...) Expand all Loading... |
34 return the output a real DRT would return for a given test, assuming that | 34 return the output a real DRT would return for a given test, assuming that |
35 test actually passes (except for reftests, which currently cause the | 35 test actually passes (except for reftests, which currently cause the |
36 MockDRT to crash). | 36 MockDRT to crash). |
37 """ | 37 """ |
38 | 38 |
39 import base64 | 39 import base64 |
40 import logging | 40 import logging |
41 import optparse | 41 import optparse |
42 import os | 42 import os |
43 import sys | 43 import sys |
| 44 import types |
44 | 45 |
45 # Since we execute this script directly as part of the unit tests, we need to en
sure | 46 # Since we execute this script directly as part of the unit tests, we need to en
sure |
46 # that Tools/Scripts is in sys.path for the next imports to work correctly. | 47 # that Tools/Scripts is in sys.path for the next imports to work correctly. |
47 script_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.
path.abspath(__file__))))) | 48 script_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.
path.abspath(__file__))))) |
48 if script_dir not in sys.path: | 49 if script_dir not in sys.path: |
49 sys.path.append(script_dir) | 50 sys.path.append(script_dir) |
50 | 51 |
| 52 from webkitpy.common import read_checksum_from_png |
51 from webkitpy.common.system.systemhost import SystemHost | 53 from webkitpy.common.system.systemhost import SystemHost |
52 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput | 54 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput |
53 from webkitpy.layout_tests.port.factory import PortFactory | 55 from webkitpy.layout_tests.port.factory import PortFactory |
54 | 56 |
55 _log = logging.getLogger(__name__) | 57 _log = logging.getLogger(__name__) |
56 | 58 |
57 | 59 |
58 class MockDRTPort(object): | 60 class MockDRTPort(object): |
59 port_name = 'mock' | 61 port_name = 'mock' |
60 | 62 |
61 @classmethod | 63 @classmethod |
62 def determine_full_port_name(cls, host, options, port_name): | 64 def determine_full_port_name(cls, host, options, port_name): |
63 return port_name | 65 return port_name |
64 | 66 |
65 def __init__(self, host, port_name, **kwargs): | 67 def __init__(self, host, port_name, **kwargs): |
66 self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''),
**kwargs) | 68 self.__delegate = PortFactory(host).get(port_name.replace('mock-', ''),
**kwargs) |
| 69 self.__delegate_driver_class = self.__delegate._driver_class |
| 70 self.__delegate._driver_class = types.MethodType(self._driver_class, sel
f.__delegate) |
67 | 71 |
68 def __getattr__(self, name): | 72 def __getattr__(self, name): |
69 return getattr(self.__delegate, name) | 73 return getattr(self.__delegate, name) |
70 | 74 |
71 def check_build(self, needs_http, printer): | 75 def check_build(self, needs_http, printer): |
72 return True | 76 return True |
73 | 77 |
74 def check_sys_deps(self, needs_http): | 78 def check_sys_deps(self, needs_http): |
75 return True | 79 return True |
76 | 80 |
77 def _driver_class(self): | 81 def _driver_class(self, delegate): |
78 return self._mocked_driver_maker | 82 return self._mocked_driver_maker |
79 | 83 |
80 @staticmethod | 84 def _mocked_driver_maker(self, port, worker_number, pixel_tests, no_timeout=
False): |
81 def _mocked_driver_maker(port, worker_number, pixel_tests, no_timeout=False)
: | 85 path_to_this_file = self.host.filesystem.abspath(__file__.replace('.pyc'
, '.py')) |
82 path_to_this_file = port.host.filesystem.abspath(__file__.replace('.pyc'
, '.py')) | 86 driver = self.__delegate_driver_class()(self, worker_number, pixel_tests
, no_timeout) |
83 driver = port.__delegate._driver_class()(port, worker_number, pixel_test
s, no_timeout) | 87 driver.cmd_line = self._overriding_cmd_line(driver.cmd_line, |
84 driver.cmd_line = port._overriding_cmd_line(driver.cmd_line, | 88 self.__delegate._path_to_dri
ver(), |
85 port.__delegate._path_to_dri
ver(), | |
86 sys.executable, | 89 sys.executable, |
87 path_to_this_file, | 90 path_to_this_file, |
88 port.__delegate.name()) | 91 self.__delegate.name()) |
89 return driver | 92 return driver |
90 | 93 |
91 @staticmethod | 94 @staticmethod |
92 def _overriding_cmd_line(original_cmd_line, driver_path, python_exe, this_fi
le, port_name): | 95 def _overriding_cmd_line(original_cmd_line, driver_path, python_exe, this_fi
le, port_name): |
93 def new_cmd_line(pixel_tests, per_test_args): | 96 def new_cmd_line(pixel_tests, per_test_args): |
94 cmd_line = original_cmd_line(pixel_tests, per_test_args) | 97 cmd_line = original_cmd_line(pixel_tests, per_test_args) |
95 index = cmd_line.index(driver_path) | 98 index = cmd_line.index(driver_path) |
96 cmd_line[index:index + 1] = [python_exe, this_file, '--platform', po
rt_name] | 99 cmd_line[index:index + 1] = [python_exe, this_file, '--platform', po
rt_name] |
97 return cmd_line | 100 return cmd_line |
98 | 101 |
(...skipping 16 matching lines...) Expand all Loading... |
115 | 118 |
116 def stop_http_server(self): | 119 def stop_http_server(self): |
117 pass | 120 pass |
118 | 121 |
119 def stop_websocket_server(self): | 122 def stop_websocket_server(self): |
120 pass | 123 pass |
121 | 124 |
122 def release_http_lock(self): | 125 def release_http_lock(self): |
123 pass | 126 pass |
124 | 127 |
125 def show_results_html_file(self, results_filename): | |
126 pass | |
127 | |
128 def _make_wdiff_available(self): | 128 def _make_wdiff_available(self): |
129 self.__delegate._wdiff_available = True | 129 self.__delegate._wdiff_available = True |
130 | 130 |
| 131 def setup_environ_for_server(self, server_name): |
| 132 env = self.__delegate.setup_environ_for_server() |
| 133 # We need to propagate PATH down so the python code can find the checkou
t. |
| 134 env['PATH'] = os.environ['PATH'] |
| 135 return env |
| 136 |
| 137 def lookup_virtual_test_args(self, test_name): |
| 138 suite = self.__delegate.lookup_virtual_suite(test_name) |
| 139 return suite.args + ['--virtual-test-suite-name', suite.name, '--virtual
-test-suite-base', suite.base] |
131 | 140 |
132 def main(argv, host, stdin, stdout, stderr): | 141 def main(argv, host, stdin, stdout, stderr): |
133 """Run the tests.""" | 142 """Run the tests.""" |
134 | 143 |
135 options, args = parse_options(argv) | 144 options, args = parse_options(argv) |
136 if options.test_shell: | 145 drt = MockDRT(options, args, host, stdin, stdout, stderr) |
137 drt = MockTestShell(options, args, host, stdin, stdout, stderr) | |
138 else: | |
139 drt = MockDRT(options, args, host, stdin, stdout, stderr) | |
140 return drt.run() | 146 return drt.run() |
141 | 147 |
142 | 148 |
143 def parse_options(argv): | 149 def parse_options(argv): |
144 # FIXME: We have to do custom arg parsing instead of using the optparse | 150 # We do custom arg parsing instead of using the optparse module |
145 # module. First, Chromium and non-Chromium DRTs have a different argument | 151 # because we don't want to have to list every command line flag DRT |
146 # syntax. Chromium uses --pixel-tests=<path>, and non-Chromium uses | 152 # accepts, and optparse complains about unrecognized flags. |
147 # --pixel-tests as a boolean flag. Second, we don't want to have to list | |
148 # every command line flag DRT accepts, but optparse complains about | |
149 # unrecognized flags. At some point it might be good to share a common | |
150 # DRT options class between this file and webkit.py and chromium.py | |
151 # just to get better type checking. | |
152 platform_index = argv.index('--platform') | |
153 platform = argv[platform_index + 1] | |
154 | 153 |
155 pixel_tests = False | 154 def get_arg(arg_name): |
156 pixel_path = None | 155 if arg_name in argv: |
157 test_shell = '--test-shell' in argv | 156 index = argv.index(arg_name) |
158 if test_shell: | 157 return argv[index + 1] |
159 for arg in argv: | 158 return None |
160 if arg.startswith('--pixel-tests'): | 159 |
161 pixel_tests = True | 160 options = optparse.Values({ |
162 pixel_path = arg[len('--pixel-tests='):] | 161 'actual_directory': get_arg('--actual-directory'), |
163 else: | 162 'platform': get_arg('--platform'), |
164 pixel_tests = '--pixel-tests' in argv | 163 'virtual_test_suite_base': get_arg('--virtual-test-suite-base'), |
165 options = optparse.Values({'test_shell': test_shell, 'platform': platform, '
pixel_tests': pixel_tests, 'pixel_path': pixel_path}) | 164 'virtual_test_suite_name': get_arg('--virtual-test-suite-name'), |
| 165 }) |
166 return (options, argv) | 166 return (options, argv) |
167 | 167 |
168 | 168 |
169 class MockDRT(object): | 169 class MockDRT(object): |
170 def __init__(self, options, args, host, stdin, stdout, stderr): | 170 def __init__(self, options, args, host, stdin, stdout, stderr): |
171 self._options = options | 171 self._options = options |
172 self._args = args | 172 self._args = args |
173 self._host = host | 173 self._host = host |
174 self._stdout = stdout | 174 self._stdout = stdout |
175 self._stdin = stdin | 175 self._stdin = stdin |
(...skipping 12 matching lines...) Expand all Loading... |
188 return 0 | 188 return 0 |
189 driver_input = self.input_from_line(line) | 189 driver_input = self.input_from_line(line) |
190 dirname, basename = self._port.split_test(driver_input.test_name) | 190 dirname, basename = self._port.split_test(driver_input.test_name) |
191 is_reftest = (self._port.reference_files(driver_input.test_name) or | 191 is_reftest = (self._port.reference_files(driver_input.test_name) or |
192 self._port.is_reference_html_file(self._port._filesyst
em, dirname, basename)) | 192 self._port.is_reference_html_file(self._port._filesyst
em, dirname, basename)) |
193 output = self.output_for_test(driver_input, is_reftest) | 193 output = self.output_for_test(driver_input, is_reftest) |
194 self.write_test_output(driver_input, output, is_reftest) | 194 self.write_test_output(driver_input, output, is_reftest) |
195 | 195 |
196 def input_from_line(self, line): | 196 def input_from_line(self, line): |
197 vals = line.strip().split("'") | 197 vals = line.strip().split("'") |
198 if len(vals) == 1: | 198 uri = vals[0] |
199 uri = vals[0] | 199 checksum = None |
200 checksum = None | 200 should_run_pixel_tests = False |
201 else: | 201 if len(vals) == 2 and vals[1] == '--pixel-test': |
202 uri = vals[0] | 202 should_run_pixel_tests = True |
203 checksum = vals[1] | 203 elif len(vals) == 3 and vals[1] == '--pixel-test': |
| 204 should_run_pixel_tests = True |
| 205 checksum = vals[2] |
| 206 elif len(vals) != 1: |
| 207 raise NotImplementedError |
| 208 |
204 if uri.startswith('http://') or uri.startswith('https://'): | 209 if uri.startswith('http://') or uri.startswith('https://'): |
205 test_name = self._driver.uri_to_test(uri) | 210 test_name = self._driver.uri_to_test(uri) |
206 else: | 211 else: |
207 test_name = self._port.relative_test_filename(uri) | 212 test_name = self._port.relative_test_filename(uri) |
208 | 213 |
209 return DriverInput(test_name, 0, checksum, self._options.pixel_tests) | 214 return DriverInput(test_name, 0, checksum, should_run_pixel_tests) |
210 | 215 |
211 def output_for_test(self, test_input, is_reftest): | 216 def output_for_test(self, test_input, is_reftest): |
212 port = self._port | 217 port = self._port |
| 218 if self._options.virtual_test_suite_name: |
| 219 test_input.test_name = test_input.test_name.replace(self._options.vi
rtual_test_suite_base, self._options.virtual_test_suite_name) |
213 actual_text = port.expected_text(test_input.test_name) | 220 actual_text = port.expected_text(test_input.test_name) |
214 actual_audio = port.expected_audio(test_input.test_name) | 221 actual_audio = port.expected_audio(test_input.test_name) |
215 actual_image = None | 222 actual_image = None |
216 actual_checksum = None | 223 actual_checksum = None |
217 if is_reftest: | 224 if is_reftest: |
218 # Make up some output for reftests. | 225 # Make up some output for reftests. |
219 actual_text = 'reference text\n' | 226 actual_text = 'reference text\n' |
220 actual_checksum = 'mock-checksum' | 227 actual_checksum = 'mock-checksum' |
221 actual_image = 'blank' | 228 actual_image = 'blank' |
222 if test_input.test_name.endswith('-mismatch.html'): | 229 if test_input.test_name.endswith('-mismatch.html'): |
223 actual_text = 'not reference text\n' | 230 actual_text = 'not reference text\n' |
224 actual_checksum = 'not-mock-checksum' | 231 actual_checksum = 'not-mock-checksum' |
225 actual_image = 'not blank' | 232 actual_image = 'not blank' |
226 elif self._options.pixel_tests and test_input.image_hash: | 233 elif test_input.should_run_pixel_test and test_input.image_hash: |
227 actual_checksum = port.expected_checksum(test_input.test_name) | 234 actual_checksum = port.expected_checksum(test_input.test_name) |
228 actual_image = port.expected_image(test_input.test_name) | 235 actual_image = port.expected_image(test_input.test_name) |
229 | 236 |
| 237 if self._options.actual_directory: |
| 238 actual_path = port._filesystem.join(self._options.actual_directory,
test_input.test_name) |
| 239 root, _ = port._filesystem.splitext(actual_path) |
| 240 text_path = root + '-actual.txt' |
| 241 if port._filesystem.exists(text_path): |
| 242 actual_text = port._filesystem.read_binary_file(text_path) |
| 243 audio_path = root + '-actual.wav' |
| 244 if port._filesystem.exists(audio_path): |
| 245 actual_audio = port._filesystem.read_binary_file(audio_path) |
| 246 image_path = root + '-actual.png' |
| 247 if port._filesystem.exists(image_path): |
| 248 actual_image = port._filesystem.read_binary_file(image_path) |
| 249 with port._filesystem.open_binary_file_for_reading(image_path) a
s filehandle: |
| 250 actual_checksum = read_checksum_from_png.read_checksum(fileh
andle) |
| 251 |
230 return DriverOutput(actual_text, actual_image, actual_checksum, actual_a
udio) | 252 return DriverOutput(actual_text, actual_image, actual_checksum, actual_a
udio) |
231 | 253 |
232 def write_test_output(self, test_input, output, is_reftest): | 254 def write_test_output(self, test_input, output, is_reftest): |
233 if output.audio: | 255 if output.audio: |
234 self._stdout.write('Content-Type: audio/wav\n') | 256 self._stdout.write('Content-Type: audio/wav\n') |
235 self._stdout.write('Content-Transfer-Encoding: base64\n') | 257 self._stdout.write('Content-Transfer-Encoding: base64\n') |
236 self._stdout.write(base64.b64encode(output.audio)) | 258 self._stdout.write(base64.b64encode(output.audio)) |
| 259 self._stdout.write('\n') |
237 else: | 260 else: |
238 self._stdout.write('Content-Type: text/plain\n') | 261 self._stdout.write('Content-Type: text/plain\n') |
239 # FIXME: Note that we don't ensure there is a trailing newline! | 262 # FIXME: Note that we don't ensure there is a trailing newline! |
240 # This mirrors actual (Mac) DRT behavior but is a bug. | 263 # This mirrors actual (Mac) DRT behavior but is a bug. |
241 if output.text: | 264 if output.text: |
242 self._stdout.write(output.text) | 265 self._stdout.write(output.text) |
243 | 266 |
244 self._stdout.write('#EOF\n') | 267 self._stdout.write('#EOF\n') |
245 | 268 |
246 if self._options.pixel_tests and output.image_hash: | 269 if test_input.should_run_pixel_test and output.image_hash: |
247 self._stdout.write('\n') | 270 self._stdout.write('\n') |
248 self._stdout.write('ActualHash: %s\n' % output.image_hash) | 271 self._stdout.write('ActualHash: %s\n' % output.image_hash) |
249 self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash) | 272 self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash) |
250 if output.image_hash != test_input.image_hash: | 273 if output.image_hash != test_input.image_hash: |
251 self._stdout.write('Content-Type: image/png\n') | 274 self._stdout.write('Content-Type: image/png\n') |
252 self._stdout.write('Content-Length: %s\n' % len(output.image)) | 275 self._stdout.write('Content-Length: %s\n' % len(output.image)) |
253 self._stdout.write(output.image) | 276 self._stdout.write(output.image) |
254 self._stdout.write('#EOF\n') | 277 self._stdout.write('#EOF\n') |
255 self._stdout.flush() | 278 self._stdout.flush() |
256 self._stderr.write('#EOF\n') | 279 self._stderr.write('#EOF\n') |
257 self._stderr.flush() | 280 self._stderr.flush() |
258 | 281 |
259 | 282 |
260 class MockTestShell(MockDRT): | |
261 def input_from_line(self, line): | |
262 vals = line.strip().split() | |
263 if len(vals) == 3: | |
264 uri, timeout, checksum = vals | |
265 else: | |
266 uri, timeout = vals | |
267 checksum = None | |
268 | |
269 test_name = self._driver.uri_to_test(uri) | |
270 return DriverInput(test_name, timeout, checksum, self._options.pixel_tes
ts) | |
271 | |
272 def output_for_test(self, test_input, is_reftest): | |
273 # FIXME: This is a hack to make virtual tests work. Need something more
general. | |
274 original_test_name = test_input.test_name | |
275 if '--enable-accelerated-2d-canvas' in self._args and 'canvas' in test_i
nput.test_name: | |
276 test_input.test_name = 'platform/chromium/virtual/gpu/' + test_input
.test_name | |
277 output = super(MockTestShell, self).output_for_test(test_input, is_refte
st) | |
278 test_input.test_name = original_test_name | |
279 return output | |
280 | |
281 def write_test_output(self, test_input, output, is_reftest): | |
282 self._stdout.write("#URL:%s\n" % self._driver.test_to_uri(test_input.tes
t_name)) | |
283 if self._options.pixel_tests and output.image_hash: | |
284 self._stdout.write("#MD5:%s\n" % output.image_hash) | |
285 if output.image: | |
286 self._host.filesystem.maybe_make_directory(self._host.filesystem
.dirname(self._options.pixel_path)) | |
287 self._host.filesystem.write_binary_file(self._options.pixel_path
, output.image) | |
288 if output.text: | |
289 self._stdout.write(output.text) | |
290 | |
291 if output.text and not output.text.endswith('\n'): | |
292 self._stdout.write('\n') | |
293 self._stdout.write('#EOF\n') | |
294 self._stdout.flush() | |
295 | |
296 | |
297 if __name__ == '__main__': | 283 if __name__ == '__main__': |
298 # Note that the Mock in MockDRT refers to the fact that it is emulating a | 284 # Note that the Mock in MockDRT refers to the fact that it is emulating a |
299 # real DRT, and as such, it needs access to a real SystemHost, not a MockSys
temHost. | 285 # real DRT, and as such, it needs access to a real SystemHost, not a MockSys
temHost. |
300 sys.exit(main(sys.argv[1:], SystemHost(), sys.stdin, sys.stdout, sys.stderr)
) | 286 sys.exit(main(sys.argv[1:], SystemHost(), sys.stdin, sys.stdout, sys.stderr)
) |
OLD | NEW |