OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2016 The PDFium Authors. All rights reserved. | 2 # Copyright 2016 The PDFium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import cStringIO | 6 import cStringIO |
7 import functools | 7 import functools |
8 import multiprocessing | 8 import multiprocessing |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import re | 11 import re |
12 import shutil | 12 import shutil |
13 import subprocess | 13 import subprocess |
14 import sys | 14 import sys |
15 | 15 |
16 import common | 16 import common |
| 17 import gold |
17 import pngdiffer | 18 import pngdiffer |
18 import suppressor | 19 import suppressor |
19 | 20 |
20 class KeyboardInterruptError(Exception): pass | 21 class KeyboardInterruptError(Exception): pass |
21 | 22 |
22 # Nomenclature: | 23 # Nomenclature: |
23 # x_root - "x" | 24 # x_root - "x" |
24 # x_filename - "x.ext" | 25 # x_filename - "x.ext" |
25 # x_path - "path/to/a/b/c/x.ext" | 26 # x_path - "path/to/a/b/c/x.ext" |
26 # c_dir - "path/to/a/b/c" | 27 # c_dir - "path/to/a/b/c" |
27 | 28 |
28 def TestOneFileParallel(this, test_case): | 29 def TestOneFileParallel(this, test_case): |
29 """Wrapper to call GenerateAndTest() and redirect output to stdout.""" | 30 """Wrapper to call GenerateAndTest() and redirect output to stdout.""" |
30 try: | 31 try: |
31 input_filename, source_dir = test_case | 32 input_filename, source_dir = test_case |
32 result = this.GenerateAndTest(input_filename, source_dir); | 33 result = this.GenerateAndTest(input_filename, source_dir); |
33 return (result, input_filename, source_dir) | 34 return (result, input_filename, source_dir) |
34 except KeyboardInterrupt: | 35 except KeyboardInterrupt: |
35 raise KeyboardInterruptError() | 36 raise KeyboardInterruptError() |
36 | 37 |
37 | 38 |
38 class TestRunner: | 39 class TestRunner: |
39 def __init__(self, dirname): | 40 def __init__(self, dirname): |
40 self.test_dir = dirname | 41 self.test_dir = dirname |
41 | 42 |
| 43 # GenerateAndTest returns a tuple <success, outputfiles> where |
| 44 # success is a boolean indicating whether the tests passed comparison |
| 45 # tests and outputfiles is a list tuples: |
| 46 # (path_to_image, md5_hash_of_pixelbuffer) |
42 def GenerateAndTest(self, input_filename, source_dir): | 47 def GenerateAndTest(self, input_filename, source_dir): |
43 input_root, _ = os.path.splitext(input_filename) | 48 input_root, _ = os.path.splitext(input_filename) |
44 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt') | 49 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt') |
45 | 50 |
46 pdf_path = os.path.join(self.working_dir, input_root + '.pdf') | 51 pdf_path = os.path.join(self.working_dir, input_root + '.pdf') |
47 | 52 |
48 # Remove any existing generated images from previous runs. | 53 # Remove any existing generated images from previous runs. |
49 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir, | 54 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir, |
50 self.working_dir) | 55 self.working_dir) |
51 for image in actual_images: | 56 for image in actual_images: |
52 if os.path.exists(image): | 57 if os.path.exists(image): |
53 os.remove(image) | 58 os.remove(image) |
54 | 59 |
55 sys.stdout.flush() | 60 sys.stdout.flush() |
56 | 61 |
57 raised_exception = self.Generate(source_dir, input_filename, input_root, | 62 raised_exception = self.Generate(source_dir, input_filename, input_root, |
58 pdf_path) | 63 pdf_path) |
59 | 64 |
60 if raised_exception != None: | 65 if raised_exception != None: |
61 print "FAILURE: " + input_filename + "; " + str(raised_exception) | 66 print "FAILURE: " + input_filename + "; " + str(raised_exception) |
62 return False | 67 return False, [] |
63 | 68 |
| 69 results = [] |
64 if os.path.exists(expected_txt_path): | 70 if os.path.exists(expected_txt_path): |
65 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path) | 71 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path) |
66 else: | 72 else: |
67 raised_exception = self.TestPixel(input_root, pdf_path) | 73 raised_exception, results = self.TestPixel(input_root, pdf_path) |
68 | 74 |
69 if raised_exception != None: | 75 if raised_exception != None: |
70 print "FAILURE: " + input_filename + "; " + str(raised_exception) | 76 print "FAILURE: " + input_filename + "; " + str(raised_exception) |
71 return False | 77 return False, results |
72 | 78 |
73 if len(actual_images): | 79 if len(actual_images): |
74 if self.image_differ.HasDifferences(input_filename, source_dir, | 80 if self.image_differ.HasDifferences(input_filename, source_dir, |
75 self.working_dir): | 81 self.working_dir): |
76 return False | 82 return False, results |
77 | 83 return True, results |
78 return True | |
79 | 84 |
80 def Generate(self, source_dir, input_filename, input_root, pdf_path): | 85 def Generate(self, source_dir, input_filename, input_root, pdf_path): |
81 original_path = os.path.join(source_dir, input_filename) | 86 original_path = os.path.join(source_dir, input_filename) |
82 input_path = os.path.join(source_dir, input_root + '.in') | 87 input_path = os.path.join(source_dir, input_root + '.in') |
83 | 88 |
84 input_event_path = os.path.join(source_dir, input_root + ".evt") | 89 input_event_path = os.path.join(source_dir, input_root + ".evt") |
85 if os.path.exists(input_event_path): | 90 if os.path.exists(input_event_path): |
86 output_event_path = os.path.splitext(pdf_path)[0] + ".evt" | 91 output_event_path = os.path.splitext(pdf_path)[0] + ".evt" |
87 shutil.copyfile(input_event_path, output_event_path) | 92 shutil.copyfile(input_event_path, output_event_path) |
88 | 93 |
(...skipping 17 matching lines...) Expand all Loading... |
106 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) | 111 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) |
107 cmd_to_run.extend([self.pdfium_test_path, pdf_path]) | 112 cmd_to_run.extend([self.pdfium_test_path, pdf_path]) |
108 subprocess.check_call(cmd_to_run, stdout=outfile) | 113 subprocess.check_call(cmd_to_run, stdout=outfile) |
109 | 114 |
110 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] | 115 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] |
111 return common.RunCommand(cmd) | 116 return common.RunCommand(cmd) |
112 | 117 |
113 | 118 |
114 def TestPixel(self, input_root, pdf_path): | 119 def TestPixel(self, input_root, pdf_path): |
115 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) | 120 cmd_to_run = common.DrMemoryWrapper(self.drmem_wrapper, input_root) |
116 cmd_to_run.extend([self.pdfium_test_path, '--send-events', '--png', | 121 cmd_to_run.extend([self.pdfium_test_path, '--send-events', '--png']) |
117 pdf_path]) | 122 if self.gold_results: |
118 return common.RunCommand(cmd_to_run) | 123 cmd_to_run.append('--md5') |
119 | 124 cmd_to_run.append(pdf_path) |
| 125 return common.RunCommandExtractHashedFiles(cmd_to_run) |
120 | 126 |
121 def HandleResult(self, input_filename, input_path, result): | 127 def HandleResult(self, input_filename, input_path, result): |
| 128 if self.gold_results: |
| 129 success, image_paths = result |
| 130 for img_path, md5_hash in image_paths: |
| 131 # the output filename (without extension becomes the test name) |
| 132 test_name = os.path.splitext(os.path.split(img_path)[1])[0] |
| 133 self.gold_results.AddTestResult(test_name, md5_hash, img_path) |
| 134 |
122 if self.test_suppressor.IsResultSuppressed(input_filename): | 135 if self.test_suppressor.IsResultSuppressed(input_filename): |
123 if result: | 136 if result: |
124 self.surprises.append(input_path) | 137 self.surprises.append(input_path) |
125 else: | 138 else: |
126 if not result: | 139 if not result: |
127 self.failures.append(input_path) | 140 self.failures.append(input_path) |
128 | 141 |
129 | 142 |
130 def Run(self): | 143 def Run(self): |
131 parser = optparse.OptionParser() | 144 parser = optparse.OptionParser() |
| 145 |
132 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), | 146 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), |
133 help='relative path from the base source directory') | 147 help='relative path from the base source directory') |
| 148 |
134 parser.add_option('-j', default=multiprocessing.cpu_count(), | 149 parser.add_option('-j', default=multiprocessing.cpu_count(), |
135 dest='num_workers', type='int', | 150 dest='num_workers', type='int', |
136 help='run NUM_WORKERS jobs in parallel') | 151 help='run NUM_WORKERS jobs in parallel') |
| 152 |
137 parser.add_option('--wrapper', default='', dest="wrapper", | 153 parser.add_option('--wrapper', default='', dest="wrapper", |
138 help='wrapper for running test under Dr. Memory') | 154 help='wrapper for running test under Dr. Memory') |
| 155 |
| 156 parser.add_option('--gold_properties', default='', dest="gold_properties", |
| 157 help='Key value pairs that are written to the top level of
the JSON file that is ingested by Gold.') |
| 158 |
| 159 parser.add_option('--gold_key', default='', dest="gold_key", |
| 160 help='Key value pairs that are added to the "key" field of
the JSON file that is ingested by Gold.') |
| 161 |
| 162 parser.add_option('--gold_output_dir', default='', dest="gold_output_dir", |
| 163 help='Path of where to write the JSON output to be uploade
d to Gold.') |
| 164 |
| 165 parser.add_option('--ignore_errors', action="store_true", dest="ignore_error
s", |
| 166 help='Prevents the return value from being non-zero when i
mage comparison fails.') |
| 167 |
139 options, args = parser.parse_args() | 168 options, args = parser.parse_args() |
140 | 169 |
141 finder = common.DirectoryFinder(options.build_dir) | 170 finder = common.DirectoryFinder(options.build_dir) |
142 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') | 171 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') |
143 self.text_diff_path = finder.ScriptPath('text_diff.py') | 172 self.text_diff_path = finder.ScriptPath('text_diff.py') |
144 | 173 |
145 self.drmem_wrapper = options.wrapper | 174 self.drmem_wrapper = options.wrapper |
146 | 175 |
147 self.source_dir = finder.TestingDir() | 176 self.source_dir = finder.TestingDir() |
148 if self.test_dir != 'corpus': | 177 if self.test_dir != 'corpus': |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
184 for input_filename in filename_list: | 213 for input_filename in filename_list: |
185 if input_file_re.match(input_filename): | 214 if input_file_re.match(input_filename): |
186 input_path = os.path.join(file_dir, input_filename) | 215 input_path = os.path.join(file_dir, input_filename) |
187 if not self.test_suppressor.IsExecutionSuppressed(input_path): | 216 if not self.test_suppressor.IsExecutionSuppressed(input_path): |
188 if os.path.isfile(input_path): | 217 if os.path.isfile(input_path): |
189 test_cases.append((input_filename, file_dir)) | 218 test_cases.append((input_filename, file_dir)) |
190 | 219 |
191 self.failures = [] | 220 self.failures = [] |
192 self.surprises = [] | 221 self.surprises = [] |
193 | 222 |
| 223 # Collect Gold results if an output directory was named. |
| 224 self.gold_results = None |
| 225 if options.gold_output_dir: |
| 226 self.gold_results = gold.GoldResults("pdfium", |
| 227 options.gold_output_dir, |
| 228 options.gold_properties, |
| 229 options.gold_key) |
| 230 |
194 if options.num_workers > 1 and len(test_cases) > 1: | 231 if options.num_workers > 1 and len(test_cases) > 1: |
195 try: | 232 try: |
196 pool = multiprocessing.Pool(options.num_workers) | 233 pool = multiprocessing.Pool(options.num_workers) |
197 worker_func = functools.partial(TestOneFileParallel, self) | 234 worker_func = functools.partial(TestOneFileParallel, self) |
198 | 235 |
199 worker_results = pool.imap(worker_func, test_cases) | 236 worker_results = pool.imap(worker_func, test_cases) |
200 for worker_result in worker_results: | 237 for worker_result in worker_results: |
201 result, input_filename, source_dir = worker_result | 238 result, input_filename, source_dir = worker_result |
202 input_path = os.path.join(source_dir, input_filename) | 239 input_path = os.path.join(source_dir, input_filename) |
203 | 240 |
204 self.HandleResult(input_filename, input_path, result) | 241 self.HandleResult(input_filename, input_path, result) |
205 | 242 |
206 except KeyboardInterrupt: | 243 except KeyboardInterrupt: |
207 pool.terminate() | 244 pool.terminate() |
208 finally: | 245 finally: |
209 pool.close() | 246 pool.close() |
210 pool.join() | 247 pool.join() |
211 else: | 248 else: |
212 for test_case in test_cases: | 249 for test_case in test_cases: |
213 input_filename, input_file_dir = test_case | 250 input_filename, input_file_dir = test_case |
214 result = self.GenerateAndTest(input_filename, input_file_dir) | 251 result = self.GenerateAndTest(input_filename, input_file_dir) |
215 self.HandleResult(input_filename, | 252 self.HandleResult(input_filename, |
216 os.path.join(input_file_dir, input_filename), result) | 253 os.path.join(input_file_dir, input_filename), result) |
217 | 254 |
| 255 if self.gold_results: |
| 256 self.gold_results.WriteResults() |
| 257 |
218 if self.surprises: | 258 if self.surprises: |
219 self.surprises.sort() | 259 self.surprises.sort() |
220 print '\n\nUnexpected Successes:' | 260 print '\n\nUnexpected Successes:' |
221 for surprise in self.surprises: | 261 for surprise in self.surprises: |
222 print surprise; | 262 print surprise; |
223 | 263 |
224 if self.failures: | 264 if self.failures: |
225 self.failures.sort() | 265 self.failures.sort() |
226 print '\n\nSummary of Failures:' | 266 print '\n\nSummary of Failures:' |
227 for failure in self.failures: | 267 for failure in self.failures: |
228 print failure | 268 print failure |
229 return 1 | 269 if not options.ignore_errors: |
230 | 270 return 1 |
231 return 0 | 271 return 0 |
OLD | NEW |