OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2016 The PDFium Authors. All rights reserved. | 2 # Copyright 2016 The PDFium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import cStringIO | 6 import cStringIO |
7 import functools | 7 import functools |
8 import multiprocessing | 8 import multiprocessing |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import re | 11 import re |
12 import shutil | 12 import shutil |
13 import subprocess | 13 import subprocess |
14 import sys | 14 import sys |
15 | 15 |
16 import common | 16 import common |
17 import gold | |
17 import pngdiffer | 18 import pngdiffer |
18 import suppressor | 19 import suppressor |
19 | 20 |
20 class KeyboardInterruptError(Exception): pass | 21 class KeyboardInterruptError(Exception): pass |
21 | 22 |
22 # Nomenclature: | 23 # Nomenclature: |
23 # x_root - "x" | 24 # x_root - "x" |
24 # x_filename - "x.ext" | 25 # x_filename - "x.ext" |
25 # x_path - "path/to/a/b/c/x.ext" | 26 # x_path - "path/to/a/b/c/x.ext" |
26 # c_dir - "path/to/a/b/c" | 27 # c_dir - "path/to/a/b/c" |
27 | 28 |
28 def TestOneFileParallel(this, test_case): | 29 def TestOneFileParallel(this, test_case): |
29 """Wrapper to call GenerateAndTest() and redirect output to stdout.""" | 30 """Wrapper to call GenerateAndTest() and redirect output to stdout.""" |
30 try: | 31 try: |
31 input_filename, source_dir = test_case | 32 input_filename, source_dir = test_case |
32 result = this.GenerateAndTest(input_filename, source_dir); | 33 result = this.GenerateAndTest(input_filename, source_dir); |
33 return (result, input_filename, source_dir) | 34 return (result, input_filename, source_dir) |
34 except KeyboardInterrupt: | 35 except KeyboardInterrupt: |
35 raise KeyboardInterruptError() | 36 raise KeyboardInterruptError() |
36 | 37 |
37 | 38 |
38 class TestRunner: | 39 class TestRunner: |
39 def __init__(self, dirname): | 40 def __init__(self, dirname): |
40 self.test_dir = dirname | 41 self.test_dir = dirname |
41 | 42 |
43 # GenerateAndTest returns a tuple <success, outputfiles> where | |
44 # success is a boolean indicating whether the tests passed comparison | |
45 # tests and outputfiles is a list tuples: | |
46 # (path_to_image, md5_hash_of_pixelbuffer) | |
42 def GenerateAndTest(self, input_filename, source_dir): | 47 def GenerateAndTest(self, input_filename, source_dir): |
43 input_root, _ = os.path.splitext(input_filename) | 48 input_root, _ = os.path.splitext(input_filename) |
44 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt') | 49 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt') |
45 | 50 |
46 pdf_path = os.path.join(self.working_dir, input_root + '.pdf') | 51 pdf_path = os.path.join(self.working_dir, input_root + '.pdf') |
47 | 52 |
48 # Remove any existing generated images from previous runs. | 53 # Remove any existing generated images from previous runs. |
49 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir, | 54 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir, |
50 self.working_dir) | 55 self.working_dir) |
51 for image in actual_images: | 56 for image in actual_images: |
52 if os.path.exists(image): | 57 if os.path.exists(image): |
53 os.remove(image) | 58 os.remove(image) |
54 | 59 |
55 sys.stdout.flush() | 60 sys.stdout.flush() |
56 | 61 |
57 raised_exception = self.Generate(source_dir, input_filename, input_root, | 62 raised_exception = self.Generate(source_dir, input_filename, input_root, |
58 pdf_path) | 63 pdf_path) |
59 | 64 |
60 if raised_exception != None: | 65 if raised_exception != None: |
61 print "FAILURE: " + input_filename + "; " + str(raised_exception) | 66 print "FAILURE: " + input_filename + "; " + str(raised_exception) |
62 return False | 67 return False, [] |
63 | 68 |
69 results = [] | |
64 if os.path.exists(expected_txt_path): | 70 if os.path.exists(expected_txt_path): |
65 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path) | 71 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path) |
66 else: | 72 else: |
67 raised_exception = self.TestPixel(input_root, pdf_path) | 73 raised_exception, results = self.TestPixel(input_root, pdf_path) |
68 | 74 |
69 if raised_exception != None: | 75 if raised_exception != None: |
70 print "FAILURE: " + input_filename + "; " + str(raised_exception) | 76 print "FAILURE: " + input_filename + "; " + str(raised_exception) |
71 return False | 77 return False, results |
72 | 78 |
73 if len(actual_images): | 79 if len(actual_images): |
74 if self.image_differ.HasDifferences(input_filename, source_dir, | 80 if self.image_differ.HasDifferences(input_filename, source_dir, |
75 self.working_dir): | 81 self.working_dir): |
76 return False | 82 return False, results |
77 | 83 return True, results |
78 return True | |
79 | 84 |
80 def Generate(self, source_dir, input_filename, input_root, pdf_path): | 85 def Generate(self, source_dir, input_filename, input_root, pdf_path): |
81 original_path = os.path.join(source_dir, input_filename) | 86 original_path = os.path.join(source_dir, input_filename) |
82 input_path = os.path.join(source_dir, input_root + '.in') | 87 input_path = os.path.join(source_dir, input_root + '.in') |
83 | 88 |
84 input_event_path = os.path.join(source_dir, input_root + ".evt") | 89 input_event_path = os.path.join(source_dir, input_root + ".evt") |
85 if os.path.exists(input_event_path): | 90 if os.path.exists(input_event_path): |
86 output_event_path = os.path.splitext(pdf_path)[0] + ".evt" | 91 output_event_path = os.path.splitext(pdf_path)[0] + ".evt" |
87 shutil.copyfile(input_event_path, output_event_path) | 92 shutil.copyfile(input_event_path, output_event_path) |
88 | 93 |
(...skipping 17 matching lines...) Expand all Loading... | |
106 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) | 111 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) |
107 cmd_to_run.extend([self.pdfium_test_path, pdf_path]) | 112 cmd_to_run.extend([self.pdfium_test_path, pdf_path]) |
108 subprocess.check_call(cmd_to_run, stdout=outfile) | 113 subprocess.check_call(cmd_to_run, stdout=outfile) |
109 | 114 |
110 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] | 115 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] |
111 return common.RunCommand(cmd) | 116 return common.RunCommand(cmd) |
112 | 117 |
113 | 118 |
114 def TestPixel(self, input_root, pdf_path): | 119 def TestPixel(self, input_root, pdf_path): |
115 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) | 120 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) |
116 cmd_to_run.extend([self.pdfium_test_path, '--send-events', '--png', | 121 cmd_to_run.extend([self.pdfium_test_path, '--send-events', '--png']) |
117 pdf_path]) | 122 if self.gold_results: |
118 return common.RunCommand(cmd_to_run) | 123 cmd_to_run.append('--md5') |
119 | 124 cmd_to_run.append(pdf_path) |
125 return common.RunCommandExtractHashedFiles(cmd_to_run) | |
120 | 126 |
121 def HandleResult(self, input_filename, input_path, result): | 127 def HandleResult(self, input_filename, input_path, result): |
128 if self.gold_results: | |
129 success, image_paths = result | |
130 for img_path, md5_hash in image_paths: | |
131 # the output filename (without extension becomes the test name) | |
132 test_name = os.path.splitext(os.path.split(img_path)[1])[0] | |
133 self.gold_results.AddTestResult(test_name, md5_hash, img_path) | |
134 | |
122 if self.test_suppressor.IsResultSuppressed(input_filename): | 135 if self.test_suppressor.IsResultSuppressed(input_filename): |
123 if result: | 136 if result: |
124 self.surprises.append(input_path) | 137 self.surprises.append(input_path) |
125 else: | 138 else: |
126 if not result: | 139 if not result: |
127 self.failures.append(input_path) | 140 self.failures.append(input_path) |
128 | 141 |
129 | 142 |
130 def Run(self): | 143 def Run(self): |
131 parser = optparse.OptionParser() | 144 parser = optparse.OptionParser() |
132 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), | 145 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), |
133 help='relative path from the base source directory') | 146 help='relative path from the base source directory') |
134 parser.add_option('-j', default=multiprocessing.cpu_count(), | 147 parser.add_option('-j', default=multiprocessing.cpu_count(), |
135 dest='num_workers', type='int', | 148 dest='num_workers', type='int', |
136 help='run NUM_WORKERS jobs in parallel') | 149 help='run NUM_WORKERS jobs in parallel') |
137 parser.add_option('--wrapper', default='', dest="wrapper", | 150 parser.add_option('--wrapper', default='', dest="wrapper", |
138 help='wrapper for running test under Dr. Memory') | 151 help='wrapper for running test under Dr. Memory') |
152 parser.add_option('--gold_properties', default='', dest="gold_properties", | |
153 help='Key value pairs that are written to the top level of the JSON file that is ingested by Gold.') | |
borenet
2016/12/21 19:47:29
Nit: Line break.
stephana
2016/12/21 21:47:58
Done.
| |
154 parser.add_option('--gold_key', default='', dest="gold_key", | |
155 help='Key value pairs that are added to the "key" field of the JSON file that is ingested by Gold.') | |
156 parser.add_option('--gold_output_dir', default='', dest="gold_output_dir", | |
157 help='Path of where to write the JSON output to be uploade d to Gold.') | |
borenet
2016/12/21 19:47:28
Nit: Line break.
stephana
2016/12/21 21:47:58
Done.
| |
158 parser.add_option('--ignore_errors', action="store_true", dest="ignore_error s", | |
159 help='Prevents the return value from being non-zero when i mage comparison fails.') | |
borenet
2016/12/21 19:47:28
Nit: Line break.
stephana
2016/12/21 21:47:58
Done.
| |
139 options, args = parser.parse_args() | 160 options, args = parser.parse_args() |
140 | 161 |
141 finder = common.DirectoryFinder(options.build_dir) | 162 finder = common.DirectoryFinder(options.build_dir) |
142 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') | 163 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') |
143 self.text_diff_path = finder.ScriptPath('text_diff.py') | 164 self.text_diff_path = finder.ScriptPath('text_diff.py') |
144 | 165 |
145 self.drmem_wrapper = options.wrapper | 166 self.drmem_wrapper = options.wrapper |
146 | 167 |
147 self.source_dir = finder.TestingDir() | 168 self.source_dir = finder.TestingDir() |
148 if self.test_dir != 'corpus': | 169 if self.test_dir != 'corpus': |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
184 for input_filename in filename_list: | 205 for input_filename in filename_list: |
185 if input_file_re.match(input_filename): | 206 if input_file_re.match(input_filename): |
186 input_path = os.path.join(file_dir, input_filename) | 207 input_path = os.path.join(file_dir, input_filename) |
187 if not self.test_suppressor.IsExecutionSuppressed(input_path): | 208 if not self.test_suppressor.IsExecutionSuppressed(input_path): |
188 if os.path.isfile(input_path): | 209 if os.path.isfile(input_path): |
189 test_cases.append((input_filename, file_dir)) | 210 test_cases.append((input_filename, file_dir)) |
190 | 211 |
191 self.failures = [] | 212 self.failures = [] |
192 self.surprises = [] | 213 self.surprises = [] |
193 | 214 |
215 # Collect Gold results if an output directory was named. | |
216 self.gold_results = None | |
217 if options.gold_output_dir: | |
218 self.gold_results = gold.GoldResults("pdfium", | |
219 options.gold_output_dir, | |
220 options.gold_properties, | |
221 options.gold_key) | |
222 | |
194 if options.num_workers > 1 and len(test_cases) > 1: | 223 if options.num_workers > 1 and len(test_cases) > 1: |
195 try: | 224 try: |
196 pool = multiprocessing.Pool(options.num_workers) | 225 pool = multiprocessing.Pool(options.num_workers) |
197 worker_func = functools.partial(TestOneFileParallel, self) | 226 worker_func = functools.partial(TestOneFileParallel, self) |
198 | 227 |
199 worker_results = pool.imap(worker_func, test_cases) | 228 worker_results = pool.imap(worker_func, test_cases) |
200 for worker_result in worker_results: | 229 for worker_result in worker_results: |
201 result, input_filename, source_dir = worker_result | 230 result, input_filename, source_dir = worker_result |
202 input_path = os.path.join(source_dir, input_filename) | 231 input_path = os.path.join(source_dir, input_filename) |
203 | 232 |
204 self.HandleResult(input_filename, input_path, result) | 233 self.HandleResult(input_filename, input_path, result) |
205 | 234 |
206 except KeyboardInterrupt: | 235 except KeyboardInterrupt: |
207 pool.terminate() | 236 pool.terminate() |
208 finally: | 237 finally: |
209 pool.close() | 238 pool.close() |
210 pool.join() | 239 pool.join() |
211 else: | 240 else: |
212 for test_case in test_cases: | 241 for test_case in test_cases: |
213 input_filename, input_file_dir = test_case | 242 input_filename, input_file_dir = test_case |
214 result = self.GenerateAndTest(input_filename, input_file_dir) | 243 result = self.GenerateAndTest(input_filename, input_file_dir) |
215 self.HandleResult(input_filename, | 244 self.HandleResult(input_filename, |
216 os.path.join(input_file_dir, input_filename), result) | 245 os.path.join(input_file_dir, input_filename), result) |
217 | 246 |
247 if self.gold_results: | |
248 self.gold_results.WriteResults() | |
249 | |
218 if self.surprises: | 250 if self.surprises: |
219 self.surprises.sort() | 251 self.surprises.sort() |
220 print '\n\nUnexpected Successes:' | 252 print '\n\nUnexpected Successes:' |
221 for surprise in self.surprises: | 253 for surprise in self.surprises: |
222 print surprise; | 254 print surprise; |
223 | 255 |
224 if self.failures: | 256 if self.failures: |
225 self.failures.sort() | 257 self.failures.sort() |
226 print '\n\nSummary of Failures:' | 258 print '\n\nSummary of Failures:' |
227 for failure in self.failures: | 259 for failure in self.failures: |
228 print failure | 260 print failure |
229 return 1 | 261 if not options.ignore_errors: |
230 | 262 return 1 |
231 return 0 | 263 return 0 |
OLD | NEW |