| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/python | |
| 2 # Copyright (c) 2012 The Native Client Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 import collections | |
| 7 import glob | |
| 8 import optparse | |
| 9 import re | |
| 10 import subprocess | |
| 11 import sys | |
| 12 | |
| 13 import test_format | |
| 14 | |
| 15 | |
| 16 NCVAL_VERDICT = { | |
| 17 '*** <input> is safe ***': True, | |
| 18 '*** <input> IS UNSAFE ***': False} | |
| 19 | |
| 20 | |
| 21 ValidatorResult = collections.namedtuple('ValidatorResult', 'verdict offsets') | |
| 22 | |
| 23 | |
| 24 def ParseNval(nval_content): | |
| 25 """Parse content of @nval section. | |
| 26 | |
| 27 Args: | |
| 28 nval_content: Content of @nval section (as produced by 32-bit ncval | |
| 29 with options --max_errors=-1 --detailed=false --cpuid-all). | |
| 30 | |
| 31 Returns: | |
| 32 ValidatorResult | |
| 33 """ | |
| 34 | |
| 35 lines = nval_content.split('\n') | |
| 36 last_line = lines.pop() | |
| 37 assert last_line == '' | |
| 38 verdict = NCVAL_VERDICT[lines.pop()] | |
| 39 | |
| 40 offsets = set() | |
| 41 for line in lines: | |
| 42 if re.match(r'.+ > .+ \(read overflow of .+ bytes\)', line): | |
| 43 # Add to offsets something that designedly can't occur in offsets | |
| 44 # produced by ParseRdfaOutput, so that difference will show up and | |
| 45 # corresponding test won't be missed. | |
| 46 offsets.add('read overflow') | |
| 47 continue | |
| 48 if line == 'ErrorSegmentation': | |
| 49 # Same here. | |
| 50 offsets.add('ErrorSegmentation') | |
| 51 continue | |
| 52 | |
| 53 # Parse error message of the form | |
| 54 # VALIDATOR: 4: Bad prefix usage | |
| 55 m = re.match(r'VALIDATOR: ([0-9a-f]+): (.*)$', line, re.IGNORECASE) | |
| 56 | |
| 57 assert m is not None, "can't parse %r" % line | |
| 58 | |
| 59 offset = int(m.group(1), 16) | |
| 60 offsets.add(offset) | |
| 61 | |
| 62 return ValidatorResult(verdict=verdict, offsets=offsets) | |
| 63 | |
| 64 | |
| 65 def ParseRval(rval_content): | |
| 66 """Parse content of @rval section. | |
| 67 | |
| 68 Args: | |
| 69 nval_content: Content of @rval section (as produced by 64-bit ncval | |
| 70 with options --max_errors=-1 --readwrite_sfi --detailed=false | |
| 71 --annotate=false --cpuid-all). | |
| 72 | |
| 73 Returns: | |
| 74 ValidatorResult | |
| 75 """ | |
| 76 | |
| 77 lines = rval_content.split('\n') | |
| 78 last_line = lines.pop() | |
| 79 assert last_line == '' | |
| 80 verdict = NCVAL_VERDICT[lines.pop()] | |
| 81 | |
| 82 offsets = set() | |
| 83 for prev_line, line in zip([None] + lines, lines): | |
| 84 if line.startswith('VALIDATOR: Checking jump targets:'): | |
| 85 continue | |
| 86 if line.startswith('VALIDATOR: Checking that basic blocks are aligned'): | |
| 87 continue | |
| 88 | |
| 89 # Skip disassembler output of the form | |
| 90 # VALIDATOR: 0000000000000003: 49 89 14 07 mov [%r15+%rax*1], %rdx | |
| 91 m = re.match(r'VALIDATOR: ([0-9a-f]+):', line, re.IGNORECASE) | |
| 92 if m is not None: | |
| 93 continue | |
| 94 | |
| 95 # Parse error message of the form | |
| 96 # VALIDATOR: ERROR: 20: Bad basic block alignment. | |
| 97 m = re.match(r'VALIDATOR: ERROR: ([0-9a-f]+): (.*)', line, re.IGNORECASE) | |
| 98 if m is not None: | |
| 99 offset = int(m.group(1), 16) | |
| 100 offsets.add(offset) | |
| 101 continue | |
| 102 | |
| 103 # Parse two-line error messages of the form | |
| 104 # VALIDATOR: 0000000000000003: 49 89 14 07 mov [%r15+%rax*1], %rdx | |
| 105 # VALIDATOR: ERROR: Invalid index register in memory offset | |
| 106 m = re.match(r'VALIDATOR: (ERROR|WARNING): .*$', line, re.IGNORECASE) | |
| 107 if m is not None: | |
| 108 message_type = m.group(1) | |
| 109 assert prev_line is not None, ( | |
| 110 "can't deduce error offset because line %r " | |
| 111 "is not preceded with disassembly" % line) | |
| 112 m2 = re.match(r'VALIDATOR: ([0-9a-f]+):', prev_line, re.IGNORECASE) | |
| 113 assert m2 is not None, "can't parse line %r preceding line %r" % ( | |
| 114 prev_line, | |
| 115 line) | |
| 116 offset = int(m2.group(1), 16) | |
| 117 if message_type != 'WARNING': | |
| 118 offsets.add(offset) | |
| 119 continue | |
| 120 | |
| 121 raise AssertionError("can't parse line %r" % line) | |
| 122 | |
| 123 return ValidatorResult(verdict=verdict, offsets=offsets) | |
| 124 | |
| 125 | |
| 126 RDFA_VERDICT = { | |
| 127 'return code: 0': True, | |
| 128 'return code: 1': False} | |
| 129 | |
| 130 | |
| 131 def ParseRdfaOutput(rdfa_content): | |
| 132 """Parse content of @rdfa_output section. | |
| 133 | |
| 134 Args: | |
| 135 rdfa_content: Content of @rdfa_output section in .test file. | |
| 136 | |
| 137 Returns: | |
| 138 ValidatorResult | |
| 139 """ | |
| 140 | |
| 141 lines = rdfa_content.split('\n') | |
| 142 assert lines[-1] == '' | |
| 143 verdict = RDFA_VERDICT[lines[-2]] | |
| 144 | |
| 145 offsets = set() | |
| 146 for line in lines[:-2]: | |
| 147 # Parse error message of the form | |
| 148 # 4: [1] DFA error in validator | |
| 149 m = re.match(r'([0-9a-f]+): \[\d+\] (.*)$', line, re.IGNORECASE) | |
| 150 assert m is not None, "can't parse %r" % line | |
| 151 offset = int(m.group(1), 16) | |
| 152 offsets.add(offset) | |
| 153 | |
| 154 return ValidatorResult(verdict=verdict, offsets=offsets) | |
| 155 | |
| 156 | |
| 157 def Compare(options, items_list, stats): | |
| 158 val_field = {32: 'nval', 64: 'rval'}[options.bits] | |
| 159 val_parser = {32: ParseNval, 64: ParseRval}[options.bits] | |
| 160 | |
| 161 info = dict(items_list) | |
| 162 if 'rdfa_output' not in info: | |
| 163 if val_field in info: | |
| 164 print ' rdfa_output section is missing' | |
| 165 stats['rdfa missing'] +=1 | |
| 166 else: | |
| 167 print ' both sections are missing' | |
| 168 stats['both missing'] += 1 | |
| 169 return items_list | |
| 170 if val_field not in info: | |
| 171 print ' @%s section is missing' % val_field | |
| 172 stats['val missing'] += 1 | |
| 173 return items_list | |
| 174 | |
| 175 val = val_parser(info[val_field]) | |
| 176 rdfa = ParseRdfaOutput(info['rdfa_output']) | |
| 177 | |
| 178 if val == rdfa: | |
| 179 stats['agree'] += 1 | |
| 180 if options.git: | |
| 181 # Stage the file for commit, so that files that pass can be | |
| 182 # committed with "git commit" (without -a or other arguments). | |
| 183 subprocess.check_call(['git', 'add', test_filename]) | |
| 184 | |
| 185 if 'validators_disagree' in info: | |
| 186 stats['spurious @validators_disagree'] += 1 | |
| 187 print (' warning: validators agree, but the section ' | |
| 188 '"@validators_disagree" is present') | |
| 189 else: | |
| 190 stats['disagree'] += 1 | |
| 191 if 'validators_disagree' in info: | |
| 192 print ' validators disagree, see @validators_disagree section' | |
| 193 else: | |
| 194 print ' validators disagree!' | |
| 195 stats['unexplained disagreements'] += 1 | |
| 196 | |
| 197 diff = ['TODO: explain this\n'] | |
| 198 if val.verdict != rdfa.verdict: | |
| 199 diff.append('old validator verdict: %s\n' % val.verdict) | |
| 200 diff.append('rdfa validator verdict: %s\n' % rdfa.verdict) | |
| 201 | |
| 202 set_diff = val.offsets - rdfa.offsets | |
| 203 if len(set_diff) > 0: | |
| 204 diff.append('errors reported by old validator but not by rdfa one:\n') | |
| 205 for offset in sorted(set_diff): | |
| 206 if isinstance(offset, int): | |
| 207 offset = hex(offset) | |
| 208 diff.append(' %s\n' % offset) | |
| 209 | |
| 210 set_diff = rdfa.offsets - val.offsets | |
| 211 if len(set_diff) > 0: | |
| 212 diff.append('errors reported by rdfa validator but not by old one:\n') | |
| 213 for offset in sorted(set_diff): | |
| 214 if isinstance(offset, int): | |
| 215 offset = hex(offset) | |
| 216 diff.append(' %s\n' % offset) | |
| 217 | |
| 218 items_list = items_list + [('validators_disagree', ''.join(diff))] | |
| 219 | |
| 220 return items_list | |
| 221 | |
| 222 | |
| 223 def main(args): | |
| 224 parser = optparse.OptionParser() | |
| 225 parser.add_option('--bits', | |
| 226 type=int, | |
| 227 help='The subarchitecture to run tests against: 32 or 64') | |
| 228 parser.add_option('--update', | |
| 229 default=False, | |
| 230 action='store_true', | |
| 231 help='When validators disagree, fill in ' | |
| 232 '@validators_disagree section (if not present)') | |
| 233 # TODO(shcherbina): Get rid of this option once most tests are committed. | |
| 234 parser.add_option('--git', | |
| 235 default=False, | |
| 236 action='store_true', | |
| 237 help='Add tests with no discrepancies to git index') | |
| 238 | |
| 239 options, args = parser.parse_args(args) | |
| 240 | |
| 241 if options.bits not in [32, 64]: | |
| 242 parser.error('specify --bits 32 or --bits 64') | |
| 243 | |
| 244 if len(args) == 0: | |
| 245 parser.error('No test files specified') | |
| 246 | |
| 247 stats = collections.defaultdict(int) | |
| 248 | |
| 249 for glob_expr in args: | |
| 250 test_files = sorted(glob.glob(glob_expr)) | |
| 251 if len(test_files) == 0: | |
| 252 raise AssertionError( | |
| 253 '%r matched no files, which was probably not intended' % glob_expr) | |
| 254 for test_file in test_files: | |
| 255 print 'Comparing', test_file | |
| 256 tests = test_format.LoadTestFile(test_file) | |
| 257 tests = [Compare(options, test, stats) for test in tests] | |
| 258 if options.update: | |
| 259 test_format.SaveTestFile(tests, test_file) | |
| 260 | |
| 261 print 'Stats:' | |
| 262 for key, value in stats.items(): | |
| 263 print ' %s: %d' %(key, value) | |
| 264 | |
| 265 if options.update: | |
| 266 if stats['unexplained disagreements'] > 0: | |
| 267 print '@validators_disagree sections were created' | |
| 268 else: | |
| 269 if stats['unexplained disagreements'] > 0: | |
| 270 sys.exit(1) | |
| 271 | |
| 272 | |
| 273 if __name__ == '__main__': | |
| 274 main(sys.argv[1:]) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 625923004:
Delete old x86 validator. (Closed)
Base URL: svn://svn.chromium.org/native_client/trunk/src/native_client