Test262-es6 test runner should handle sloppy fail better

test/test262-es6/testcfg.py

 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 15 matching lines @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 import hashlib
 import os
 import shutil
 import sys
 import tarfile
 import imp
 
+from testrunner.local import statusfile
 from testrunner.local import testsuite
 from testrunner.local import utils
 from testrunner.objects import testcase
 
 # The revision hash needs to be 7 characters?
 TEST_262_ARCHIVE_REVISION = "488c0a7"  # This is the 2015-06-11 revision.
 TEST_262_ARCHIVE_MD5 = "f7d4ec9be81f1e1f10fd8a61c71baead"
 TEST_262_URL = "https://github.com/tc39/test262/tarball/%s"
 TEST_262_HARNESS_FILES = ["sta.js", "assert.js"]
 
@@ skipping 83 matching lines @@
 
   def IsNegativeTest(self, testcase):
     test_record = self.GetTestRecord(testcase)
     return "negative" in test_record
 
   def IsFailureOutput(self, output, testpath):
     if output.exit_code != 0:
       return True
     return "FAILED!" in output.stdout
 
+  def HasUnexpectedOutput(self, testcase):
+    outcome = self.GetOutcome(testcase)
+    if (statusfile.FAIL_SLOPPY in testcase.outcomes and
+        "--use-strict" not in testcase.flags):
+      return outcome != statusfile.FAIL
+    return not outcome in (testcase.outcomes or [statusfile.PASS])
+
   def DownloadData(self):
     revision = TEST_262_ARCHIVE_REVISION
     archive_url = TEST_262_URL % revision
     archive_name = os.path.join(self.root, "tc39-test262-%s.tar.gz" % revision)
     directory_name = os.path.join(self.root, "data")
     directory_old_name = os.path.join(self.root, "data.old")
 
     # Clobber if the test is in an outdated state, i.e. if there are any other
     # archive files present.
     archive_files = [f for f in os.listdir(self.root)
@@ skipping 27 matching lines @@
         # Magic incantation to allow longer path names on Windows.
         archive.extractall(u"\\\\?\\%s" % self.root)
       else:
         archive.extractall(self.root)
       os.rename(os.path.join(self.root, "tc39-test262-%s" % revision),
                 directory_name)
 
 
 def GetSuite(name, root):
   return Test262TestSuite(name, root)