Adding a startup regression test.

tests/abi_corpus/validator_regression_test.py

Index: tests/abi_corpus/validator_regression_test.py
diff --git a/tests/abi_corpus/validator_regression_test.py b/tests/abi_corpus/validator_regression_test.py
new file mode 100755
index 0000000000000000000000000000000000000000..60388c95be31e82e57ac67ae986aad5d22297d23
--- /dev/null
+++ b/tests/abi_corpus/validator_regression_test.py
@@ -0,0 +1,148 @@
+#!/usr/bin/python
+# Copyright (c) 2012 The Native Client Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import optparse
+import os
+import subprocess
+import tempfile
+
+import corpus_utils
+
+
+KNOWN_BAD = set([
+    '1c2b051cb60367cf103128c9cd76769ffa1cf356',
+])
+
+
+class UnexpectedValidateResult(Exception):
+  pass
+
+
+def ValidateNexe(options, path, src_path, expect_pass):
+  """Run the validator on a nexe, check if the result is expected.
+
+  Args:
+    options: bag of options.
+    path: path to the nexe.
+    src_path: path to nexe on server.
+    expect_pass: boolean indicating if the nexe is expected to validate.
+  Returns:
+    True if validation matches expectations.
+  """
+  process = subprocess.Popen(
+      [options.validator, path],
+      stdout=subprocess.PIPE,
+      stderr=subprocess.PIPE)
+  process_stdout, process_stderr = process.communicate()
+  # Check if result is what we expect.
+  did_pass = (process.returncode == 0)
+  if expect_pass != did_pass:
+    if options.verbose:
+      print '-' * 70
+      print 'Validating: %s' % path
+      print 'From: %s' % src_path
+      print 'Size: %d' % os.path.getsize(path)
+      print 'SHA1: %s' % corpus_utils.Sha1FromFilename(path)
+      print 'Validator: %s' % options.validator
+      print 'Unexpected return code: %s' % process.returncode
+      print '>>> STDOUT'
+      print process_stdout
+      print '>>> STDERR'
+      print process_stderr
+      print '-' * 70
+    else:
+      print 'Failed on sha1: %s' % corpus_utils.Sha1FromFilename(path)

[Nick Bray, 2012/04/03 20:34:29]

Very optional because this is not under review: you might want to explain if
this was a false positive or a false negative.

[bradn, 2012/04/12 17:45:54]

Done.

+    return False
+  return True
+
+
+def NexeShouldValidate(path):
+  """Checks a blacklist to decide if a nexe should validate.
+
+  Args:
+    path: path to the nexe.
+  Returns:
+    Boolean indicating if the nexe should validate.
+  """
+  return corpus_utils.Sha1FromFilename(path) not in KNOWN_BAD
+
+
+def TestValidators(options, work_dir):
+  """Test x86 validators on current snapshot.
+
+  Args:
+    options: bag of options.
+    work_dir: directory to operate in.
+  """
+  nexe_filename = os.path.join(work_dir, 'test.nexe')
+  list_filename = os.path.join(work_dir, 'naclapps.list')
+  filenames = corpus_utils.DownloadNexeList(list_filename)
+  progress = corpus_utils.Progress(len(filenames))
+  for filename in filenames:
+    progress.Tally()
+    corpus_utils.PrimeCache(options, filename)
+    # Stop here if downloading only.
+    if options.download_only:
+      continue
+    # Skip if not the right architecture.
+    architecture = corpus_utils.NexeArchitecture(
+        corpus_utils.CachedPath(options, filename))
+    if architecture != options.architecture:
+      continue
+    # Validate a copy in case validator is mutating.
+    corpus_utils.CopyFromCache(options, filename, nexe_filename)
+    try:
+      result = ValidateNexe(
+          options, nexe_filename, filename,
+          NexeShouldValidate(filename))
+      progress.Result(result)
+      if not result and not options.keep_going:
+        break
+    finally:
+      try:
+        os.remove(nexe_filename)
+      except OSError:
+        print 'ERROR - unable to remove %s' % nexe_filename
+  progress.Summary(warn_only=True)
+
+
+def Main():
+  parser = optparse.OptionParser()
+  corpus_utils.SetupOptions(parser)
+  parser.add_option(
+      '--download-only', dest='download_only',
+      default=False, action='store_true',
+      help='download to cache without running the tests')
+  parser.add_option(
+      '-k', '--keep-going', dest='keep_going',
+      default=False, action='store_true',
+      help='keep going on failure')
+  parser.add_option(
+      '--validator', dest='validator',
+      help='location of validator executable')
+  parser.add_option(
+      '--arch', dest='architecture',
+      help='architecture of validator')
+  parser.add_option(
+      '-v', '--verbose', dest='verbose', default=False, action='store_true',
+      help='run in verbose mode')
+  options, args = parser.parse_args()
+  if args:
+    parser.error('unused arguments')
+  if not options.download_only:
+    if not options.validator:
+      parser.error('no validator specified')
+    if not options.architecture:
+      parser.error('no architecture specified')
+
+  work_dir = tempfile.mkdtemp(suffix='validate_nexes', prefix='tmp')
+  try:
+    TestValidators(options, work_dir)
+  finally:
+    corpus_utils.RemoveDir(work_dir)
+
+
+if __name__ == '__main__':
+  Main()