Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(8)

Side by Side Diff: testing/tools/test_runner.py

Issue 1888673003: Re-Reland Combined test runner. (Closed) Base URL: https://pdfium.googlesource.com/pdfium.git@master
Patch Set: Created 4 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « testing/tools/run_pixel_tests.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2015 The PDFium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import cStringIO
7 import functools
8 import multiprocessing
9 import optparse
10 import os
11 import re
12 import shutil
13 import subprocess
14 import sys
15
16 import common
17 import pngdiffer
18 import suppressor
19
20 class KeyboardInterruptError(Exception): pass
21
22 # Nomenclature:
23 # x_root - "x"
24 # x_filename - "x.ext"
25 # x_path - "path/to/a/b/c/x.ext"
26 # c_dir - "path/to/a/b/c"
27
28 def TestOneFileParallel(this, test_case):
29 """Wrapper to call GenerateAndTest() and redirect output to stdout."""
30 try:
31 old_stdout = sys.stdout
32 old_stderr = sys.stderr
33 sys.stdout = cStringIO.StringIO()
34 sys.stderr = sys.stdout
35
36 input_filename, source_dir = test_case
37 result = this.GenerateAndTest(input_filename, source_dir, True);
38
39 output = sys.stdout
40 sys.stdout = old_stdout
41 sys.stderr = old_stderr
42 return (result, output.getvalue(), input_filename, source_dir)
43 except KeyboardInterrupt:
44 raise KeyboardInterruptError()
45
46 class TestRunner:
47 def __init__(self, dirname):
48 self.test_dir = dirname
49
50 def GenerateAndTest(self, input_filename, source_dir, redirect_output=False):
51 input_root, _ = os.path.splitext(input_filename)
52 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt')
53
54 pdf_path = os.path.join(self.working_dir, input_root + '.pdf')
55
56 # Remove any existing generated images from previous runs.
57 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir,
58 self.working_dir)
59 for image in actual_images:
60 if os.path.exists(image):
61 os.remove(image)
62
63 sys.stdout.flush()
64
65 raised_exception = self.Generate(source_dir, input_filename, input_root,
66 pdf_path, redirect_output)
67
68 if raised_exception != None:
69 print "FAILURE: " + input_filename + "; " + str(raised_exception)
70 return False
71
72 if os.path.exists(expected_txt_path):
73 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path,
74 redirect_output)
75 else:
76 raised_exception = self.TestPixel(pdf_path, redirect_output)
77
78 if raised_exception != None:
79 print "FAILURE: " + input_filename + "; " + str(raised_exception)
80 return False
81
82 if len(actual_images):
83 if self.image_differ.HasDifferences(input_filename, source_dir,
84 self.working_dir, redirect_output):
85 return False
86
87 return True
88
89
90 def Generate(self, source_dir, input_filename, input_root, pdf_path,
91 redirect_output):
92 original_path = os.path.join(source_dir, input_filename)
93 input_path = os.path.join(source_dir, input_root + '.in')
94 txt_path = os.path.join(self.working_dir, input_root + '.txt')
95
96 if not os.path.exists(input_path):
97 if os.path.exists(original_path):
98 shutil.copyfile(original_path, pdf_path)
99 return None
100
101 sys.stdout.flush()
102 subprocess.check_call(
103 [sys.executable, self.fixup_path, '--output-dir=' + self.working_dir,
104 input_path])
105
106 with open(txt_path, 'w') as outfile:
107 # Add Dr. Memory wrapper if exists.
108 cmd = common.DrMemoryWrapper(self.drmem_wrapper, input_root)
109 cmd.extend([self.pdfium_test_path, pdf_path])
110 ret = common.RunCommandToFile(cmd, outfile)
111 return ret
112
113 def TestText(self, input_root, expected_txt_path, pdf_path, redirect_output):
114 txt_path = os.path.join(self.working_dir, input_root + '.txt')
115 with open(txt_path, 'w') as outfile:
116 subprocess.check_call([self.pdfium_test_path, pdf_path], stdout=outfile)
117
118 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path]
119 return common.RunCommand(cmd, redirect_output)
120
121
122 def TestPixel(self, pdf_path, redirect_output):
123 return common.RunCommand(
124 [self.pdfium_test_path, '--png', pdf_path], redirect_output)
125
126
127 def HandleResult(self, input_filename, input_path, result):
128 if self.test_suppressor.IsResultSuppressed(input_filename):
129 if result:
130 self.surprises.append(input_path)
131 else:
132 if not result:
133 self.failures.append(input_path)
134
135
136 def Run(self):
137 parser = optparse.OptionParser()
138 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
139 help='relative path from the base source directory')
140 parser.add_option('-j', default=multiprocessing.cpu_count(),
141 dest='num_workers', type='int',
142 help='run NUM_WORKERS jobs in parallel')
143 parser.add_option('--wrapper', default='', dest="wrapper",
144 help='wrapper for running test under Dr. Memory')
145 options, args = parser.parse_args()
146
147 finder = common.DirectoryFinder(options.build_dir)
148 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py')
149 self.text_diff_path = finder.ScriptPath('text_diff.py')
150
151 self.drmem_wrapper = options.wrapper
152
153 self.source_dir = finder.TestingDir()
154
155 if self.test_dir != 'corpus':
156 test_dir = finder.TestingDir(os.path.join('resources', self.test_dir))
157 else:
158 test_dir = finder.TestingDir(self.test_dir)
159
160 self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
161 if not os.path.exists(self.pdfium_test_path):
162 print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path
163 print "Use --build-dir to specify its location."
164 return 1
165
166 self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
167 if not os.path.exists(self.working_dir):
168 os.makedirs(self.working_dir)
169
170 self.feature_string = subprocess.check_output([self.pdfium_test_path,
171 '--show-config'])
172 self.test_suppressor = suppressor.Suppressor(finder, self.feature_string)
173 self.image_differ = pngdiffer.PNGDiffer(finder)
174
175 walk_from_dir = finder.TestingDir(test_dir);
176
177 test_cases = []
178 input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$')
179
180 if len(args):
181 for file_name in args:
182 file_name.replace(".pdf", ".in")
183 input_path = os.path.join(walk_from_dir, file_name)
184 if not os.path.isfile(input_path):
185 print "Can't find test file '%s'" % file_name
186 return 1
187
188 test_cases.append((os.path.basename(input_path),
189 os.path.dirname(input_path)))
190 else:
191 for file_dir, _, filename_list in os.walk(walk_from_dir):
192 for input_filename in filename_list:
193 if input_file_re.match(input_filename):
194 input_path = os.path.join(file_dir, input_filename)
195 if not self.test_suppressor.IsExecutionSuppressed(input_path):
196 if os.path.isfile(input_path):
197 test_cases.append((input_filename, file_dir))
198
199 self.failures = []
200 self.surprises = []
201
202 if options.num_workers > 1 and not len(args):
203 try:
204 pool = multiprocessing.Pool(options.num_workers)
205 worker_func = functools.partial(TestOneFileParallel, self)
206
207 worker_results = pool.imap(worker_func, test_cases)
208 for worker_result in worker_results:
209 result, output, input_filename, source_dir = worker_result
210 input_path = os.path.join(source_dir, input_filename)
211 sys.stdout.write(output)
212
213 self.HandleResult(input_filename, input_path, result)
214 except KeyboardInterrupt:
215 pool.terminate()
216 finally:
217 pool.close()
218 pool.join()
219 else:
220 for test_case in test_cases:
221 input_filename, input_file_dir = test_case
222 result = self.GenerateAndTest(input_filename, input_file_dir)
223 self.HandleResult(input_filename,
224 os.path.join(input_file_dir, input_filename), result)
225
226 if self.surprises:
227 self.surprises.sort()
228 print '\n\nUnexpected Successes:'
229 for surprise in self.surprises:
230 print surprise;
231
232 if self.failures:
233 self.failures.sort()
234 print '\n\nSummary of Failures:'
235 for failure in self.failures:
236 print failure
237 return 1
238
239 return 0
240
241
242
OLDNEW
« no previous file with comments | « testing/tools/run_pixel_tests.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698