Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(433)

Side by Side Diff: testing/tools/test_runner.py

Issue 1867643002: Revert of Combined test runner. (Closed) Base URL: https://pdfium.googlesource.com/pdfium.git@master
Patch Set: Created 4 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « testing/tools/run_pixel_tests.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2015 The PDFium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import cStringIO
7 import functools
8 import multiprocessing
9 import optparse
10 import os
11 import re
12 import shutil
13 import subprocess
14 import sys
15
16 import common
17 import pngdiffer
18 import suppressor
19
20 class KeyboardInterruptError(Exception): pass
21
22 # Nomenclature:
23 # x_root - "x"
24 # x_filename - "x.ext"
25 # x_path - "path/to/a/b/c/x.ext"
26 # c_dir - "path/to/a/b/c"
27
28 def TestOneFileParallel(this, test_case):
29 """Wrapper to call GenerateAndTest() and redirect output to stdout."""
30 try:
31 old_stdout = sys.stdout
32 old_stderr = sys.stderr
33 sys.stdout = cStringIO.StringIO()
34 sys.stderr = sys.stdout
35
36 input_filename, source_dir = test_case
37 result = this.GenerateAndTest(input_filename, source_dir, True);
38
39 output = sys.stdout
40 sys.stdout = old_stdout
41 sys.stderr = old_stderr
42 return (result, output.getvalue(), input_filename, source_dir)
43 except KeyboardInterrupt:
44 raise KeyboardInterruptError()
45
46 class TestRunner:
47 def __init__(self, dirname):
48 self.test_dir = dirname
49
50 def GenerateAndTest(self, input_filename, source_dir, redirect_output=False):
51 input_root, _ = os.path.splitext(input_filename)
52 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt')
53
54 pdf_path = os.path.join(self.working_dir, input_root + '.pdf')
55
56 # Remove any existing generated images from previous runs.
57 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir,
58 self.working_dir)
59 for image in actual_images:
60 if os.path.exists(image):
61 os.remove(image)
62
63 sys.stdout.flush()
64
65 raised_exception = self.Generate(source_dir, input_filename, input_root,
66 pdf_path, redirect_output)
67
68 if raised_exception != None:
69 print "FAILURE: " + input_filename + "; " + str(raised_exception)
70 return False
71
72 if os.path.exists(expected_txt_path):
73 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path,
74 redirect_output)
75 else:
76 raised_exception = self.TestPixel(pdf_path, redirect_output)
77
78 if raised_exception != None:
79 print "FAILURE: " + input_filename + "; " + str(raised_exception)
80 return False
81
82 if len(actual_images):
83 if self.image_differ.HasDifferences(input_filename, source_dir,
84 self.working_dir, redirect_output):
85 return False
86
87 return True
88
89
90 def Generate(self, source_dir, input_filename, input_root, pdf_path,
91 redirect_output):
92 original_path = os.path.join(source_dir, input_filename)
93 input_path = os.path.join(source_dir, input_root + '.in')
94
95 if not os.path.exists(input_path):
96 if os.path.exists(original_path):
97 shutil.copyfile(original_path, pdf_path)
98 return None
99
100 # Add Dr. Memory wrapper if exists.
101 cmd = common.DrMemoryWrapper(self.drmem_wrapper, input_root)
102 cmd.extend([sys.executable, self.fixup_path,
103 '--output-dir=' + self.working_dir, input_path])
104 return common.RunCommand(cmd, redirect_output)
105
106
107 def TestText(self, input_root, expected_txt_path, pdf_path, redirect_output):
108 txt_path = os.path.join(self.working_dir, input_root + '.txt')
109 with open(txt_path, 'w') as outfile:
110 subprocess.check_call([self.pdfium_test_path, pdf_path], stdout=outfile)
111
112 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path]
113 return common.RunCommand(cmd, redirect_output)
114
115
116 def TestPixel(self, pdf_path, redirect_output):
117 return common.RunCommand(
118 [self.pdfium_test_path, '--png', pdf_path], redirect_output)
119
120
121 def HandleResult(self, input_filename, input_path, result):
122 if self.test_suppressor.IsSuppressed(input_filename):
123 if result:
124 self.surprises.append(input_path)
125 else:
126 if not result:
127 self.failures.append(input_path)
128
129
130 def Run(self):
131 parser = optparse.OptionParser()
132 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
133 help='relative path from the base source directory')
134 parser.add_option('-j', default=multiprocessing.cpu_count(),
135 dest='num_workers', type='int',
136 help='run NUM_WORKERS jobs in parallel')
137 parser.add_option('--wrapper', default='', dest="wrapper",
138 help='wrapper for running test under Dr. Memory')
139 options, args = parser.parse_args()
140
141 finder = common.DirectoryFinder(options.build_dir)
142 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py')
143 self.text_diff_path = finder.ScriptPath('text_diff.py')
144
145 self.drmem_wrapper = options.wrapper
146
147 self.source_dir = finder.TestingDir()
148
149 if self.test_dir != 'corpus':
150 test_dir = finder.TestingDir(os.path.join('resources', self.test_dir))
151 else:
152 test_dir = finder.TestingDir(self.test_dir)
153
154 self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
155 if not os.path.exists(self.pdfium_test_path):
156 print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path
157 print "Use --build-dir to specify its location."
158 return 1
159
160 self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
161 if not os.path.exists(self.working_dir):
162 os.makedirs(self.working_dir)
163
164 self.feature_string = subprocess.check_output([self.pdfium_test_path,
165 '--show-config'])
166 self.test_suppressor = suppressor.Suppressor(finder, self.feature_string)
167 self.image_differ = pngdiffer.PNGDiffer(finder)
168
169 walk_from_dir = finder.TestingDir(test_dir);
170
171 test_cases = []
172 input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$')
173
174 if len(args):
175 for file_name in args:
176 file_name.replace(".pdf", ".in")
177 input_path = os.path.join(walk_from_dir, file_name)
178 if not os.path.isfile(input_path):
179 print "Can't find test file '%s'" % file_name
180 return 1
181
182 test_cases.append((os.path.basename(input_path),
183 os.path.dirname(input_path)))
184 else:
185 for file_dir, _, filename_list in os.walk(walk_from_dir):
186 for input_filename in filename_list:
187 if input_file_re.match(input_filename):
188 input_path = os.path.join(file_dir, input_filename)
189 if os.path.isfile(input_path):
190 test_cases.append((input_filename, file_dir))
191
192 self.failures = []
193 self.surprises = []
194
195 if options.num_workers > 1 and not len(args):
196 try:
197 pool = multiprocessing.Pool(options.num_workers)
198 worker_func = functools.partial(TestOneFileParallel, self)
199
200 worker_results = pool.imap(worker_func, test_cases)
201 for worker_result in worker_results:
202 result, output, input_filename, source_dir = worker_result
203 input_path = os.path.join(source_dir, input_filename)
204 sys.stdout.write(output)
205
206 self.HandleResult(input_filename, input_path, result)
207 except KeyboardInterrupt:
208 pool.terminate()
209 finally:
210 pool.close()
211 pool.join()
212 else:
213 for test_case in test_cases:
214 input_filename, input_file_dir = test_case
215 result = self.GenerateAndTest(input_filename, input_file_dir)
216 self.HandleResult(input_filename,
217 os.path.join(input_file_dir, input_filename), result)
218
219 if self.surprises:
220 self.surprises.sort()
221 print '\n\nUnexpected Successes:'
222 for surprise in self.surprises:
223 print surprise;
224
225 if self.failures:
226 self.failures.sort()
227 print '\n\nSummary of Failures:'
228 for failure in self.failures:
229 print failure
230 return 1
231
232 return 0
233
234
235
OLDNEW
« no previous file with comments | « testing/tools/run_pixel_tests.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698