(abandoned) Validator tests: convert hexes to TFs and run on bots (for prod. validator only)

src/trusted/validator/x86/testing/tf/tf.py

+# Copyright (c) 2012 The Native Client Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import copy
+import re
+
+import asm
+import utils
+import val_runner
+
+
+class Instruction(object):
+  __slots__ = [
+      'offset',
+      'asm',
+      'hex',
+      'data',
+      'outs',  # {validator: [message, ...], ...}
+  ]
+
+  def __init__(self):
+    self.offset = None
+    self.hex = None
+    self.asm = None
+    self.outs = dict((v, []) for v in val_runner.VALIDATORS)
+
+  @property
+  def size(self):
+    return len(self.data)
+
+  @property
+  def end_offset(self):
+    return self.offset + self.size
+
+  def CheckAsm(self, bits):

[Mark Seaborn, 2012/09/27 02:25:43]

This method doesn't seem to be called.

+    if self.asm is None:
+      return
+    asm_data = asm.Assemble(bits, self.asm)
+    assert self.data == asm_data, (utils.DataToReadableHex(asm_data),
+                                   utils.DataToReadableHex(self.data))
+
+  def __repr__(self):
+    return 'Instr(%s)@0x%x' % (self.asm or self.hex, self.offset)
+
+  _out_suffix = '_out'
+
+  @staticmethod
+  def Parse(bits, lines):
+    instr = Instruction()
+    for line in lines:
+      assert ':' in line, "can't parse line '%s'" % line
+      field, value = line.split(':', 1)
+      field = field.strip()
+
+      if field.endswith(Instruction._out_suffix):
+        validator = field[:-len(Instruction._out_suffix)]
+        instr.outs[validator].append(value.strip())
+      else:
+        assert getattr(instr, field) is None, 'field %s is already set' % field
+        setattr(instr, field, value.strip())
+
+    if instr.hex is not None:
+      instr.data = utils.ReadableHexToData(instr.hex)
+    else:

[Mark Seaborn, 2012/09/27 02:25:43]

Hmm, having a fallback here means that in normal testing mode (on the bots),
you're not enforcing that every section contains a "hex" field.  Ideally you
would be strict and enforce that.

+      instr.data = asm.Assemble(bits, instr.asm)
+      instr.hex = utils.DataToReadableHex(instr.data)
+    return instr
+
+  def ToLines(self, offset=None):
+    lines = []
+    if self.offset != offset:
+      lines.append('offset: %s' % self.offset)
+    if self.asm is not None:
+      lines.append('asm: %s' % self.asm)
+    if self.hex is not None:
+      lines.append('hex: %s' % self.hex)
+
+    for validator in val_runner.VALIDATORS:
+      for msg in self.outs[validator]:
+        lines.append('%s%s: %s' % (validator, Instruction._out_suffix, msg))
+
+    return lines
+
+
+class Test(object):
+  __slots__ = [
+      'bits',

[Mark Seaborn, 2012/09/27 02:25:43]

Some comments on what these fields mean would be helpful.

+      'separators',
+      'sections',
+      'instructions',
+      'safe',
+  ]
+
+  # Each separator/section is a list of strings.
+  # Number of separators always exceeds number of sections by one, because
+  # they interleave, starting and finishing with (possibly empty) separator.
+
+  @staticmethod
+  def Parse(lines):
+    test = Test()
+    test.safe = None
+    test.separators = [[]]
+    test.sections = []
+
+    # Parser state: whether we are currently parsing
+    # separator (comments/whitelines) or section.
+    in_section = False
+
+    for line in lines:
+      line = line.strip()
+      is_sep = line == '' or line.startswith('#')
+
+      if is_sep:
+        if in_section:
+          test.separators.append([line])
+        else:
+          test.separators[-1].append(line)
+      else:
+        if in_section:
+          test.sections[-1].append(line)
+        else:
+          test.sections.append([line])
+
+      in_section = not is_sep
+
+    if in_section:
+      test.separators.append([])
+
+    assert len(test.separators) == len(test.sections) + 1
+
+    # header section is required; it specifies BITS and OUTCOME
+    assert len(test.sections) >= 1
+
+    for line in test.sections[0]:
+      m = re.match(r'(.*):\s*(.*)$', line)
+      field, value = m.groups()
+      if field == 'BITS':
+        test.bits = int(value)
+      elif field == 'OUTCOME':
+        assert value in ['valid', 'invalid']
+        test.safe = value == 'valid'
+      else:
+        raise AssertionError('Unrecognized field %s in special section' %
+                             field)
+
+    test.instructions = []
+    offset = 0
+    for section in test.sections[1:]:
+      instr = Instruction.Parse(test.bits, section)
+
+      if instr.hex is None:
+        code = asm.Assemble(test.bits, instr.asm)
+        instr.hex = utils.DataToReadableHex(code)
+
+      if instr.offset is not None:
+        instr.offset = int(instr.offset)
+      else:
+        instr.offset = offset
+      test.instructions.append(instr)
+      offset = instr.end_offset
+
+    return test
+
+  def Print(self, fout):
+    self.sections[0] = ['BITS: %s' % self.bits]
+    if self.safe is not None:
+      self.sections[0].append(
+          'OUTCOME: valid' if self.safe else 'OUTCOME: invalid')
+
+    offset = 0
+    for i, instr in enumerate(self.instructions):
+      self.sections[i+1] = instr.ToLines(offset)
+      offset = instr.end_offset
+
+    assert len(self.separators) == len(self.sections) + 1
+    groups = []
+    for sep, sec in zip(self.separators, self.sections):
+      groups.append(sep)
+      groups.append(sec)
+    groups.append(self.separators[-1])
+
+    for group in groups:
+      for line in group:
+        fout.write('%s\n' % line)
+
+  def PrepareCode(self):
+    code_size = max(i.end_offset for i in self.instructions)
+    code_size = ((code_size - 1) // 32 + 1) * 32
+    code = ['\x90'] * code_size
+
+    for i in self.instructions:
+      code[i.offset : i.end_offset] = list(i.data)
+
+    return ''.join(code)
+
+  def RunValidator(self, validator):
+    assert validator in val_runner.VALIDATORS
+    if validator == 'nc':
+      return val_runner.RunValidator(validator, self.bits, self.PrepareCode())
+
+    test = copy.deepcopy(self)
+
+    # When RDFA validator encounters invalid instruction, it stops processing
+    # current 32-byte bundle and moves on straight to the next one, while
+    # prod. validator recovers from error and continues decoding.
+    # To avoid many spurious errors, some kind of error recovery for RDFA
+    # is emulated in the following process:
+    #
+    # Whenever instuction can't be decoded, the whole section containing it
+    # is replaced with nops, and validation is repeated from start. This
+    # patching can happen several times. As a result of validation we take
+    # union of all errors discovered on all passes.
+    # This approach is expected to work well only when partition to sections
+    # matches closely partition to instructions, which is reasonable assumption
+    # (typical section is a single instruction or few instructions forming
+    # pseudoinstruction).
+    #
+    # This procedure is more or less safe (in a sense that it's unlikely
+    # to mask any bugs in validator), because first it is guaranteed to
+    # preserve all errors found in the first pass (on unmodified code),
+    # and second all spurious errors incorrectly introduced by patching
+    # will likely be spotted as discrepancies compared to prod. validator.

[Mark Seaborn, 2012/09/27 02:25:43]

This means you're relying on having the prod validator around for testing
purposes.  But when it goes away, doesn't it leave this NOP-patching strategy on
shakier ground?

+
+    errors = set()
+    safe = True
+
+    while True:
+      res = val_runner.RunValidator(validator, test.bits, test.PrepareCode())
+      safe = safe and res.safe
+      errors.update(res.errors)
+
+      patched = False
+      for loc, msg in res.errors:
+        if msg == 'DFA error in validator':
+          for i in test.instructions:
+            if i.offset == loc:
+              nops = '\x90' * i.size
+              if i.data != nops:
+                i.data = nops
+                patched = True
+      if not patched:
+        break
+
+    return val_runner.ValidationResults(safe=safe, errors=sorted(errors))
+
+  def ExpectedErrors(self, validator):
+    errors = []
+    for i in self.instructions:
+      for msg in i.outs[validator]:
+        # Check if message has the form
+        #   [at +<delta>] <error message>
+        m = re.match(r'\[at \+(\d+)\]\s(.*)$', msg)
+        if m is not None:
+          delta = int(m.group(1))
+          msg = m.group(2)
+        else:
+          delta = 0
+        errors.append((i.offset + delta, msg))
+    return errors