[test] Process to locally develop and upstream test262 tests

test/test262/testcfg.py

 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
 #       with the distribution.
 #     * Neither the name of Google Inc. nor the names of its
 #       contributors may be used to endorse or promote products derived
 #       from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 import imp
+import itertools
 import os
 import re
 import sys
 import tarfile
 
 
 from testrunner.local import statusfile
 from testrunner.local import testsuite
 from testrunner.local import utils
 from testrunner.objects import testcase
 
 DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
 ARCHIVE = DATA + ".tar"
 
 TEST_262_HARNESS_FILES = ["sta.js", "assert.js"]
 TEST_262_NATIVE_FILES = ["detachArrayBuffer.js"]
 
 TEST_262_SUITE_PATH = ["data", "test"]
 TEST_262_HARNESS_PATH = ["data", "harness"]
 TEST_262_TOOLS_PATH = ["harness", "src"]
+TEST_262_LOCAL_TESTS_PATH = ["local-tests", "test"]
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                              *TEST_262_TOOLS_PATH))
 
 ALL_VARIANT_FLAGS_STRICT = dict(
     (v, [flags + ["--use-strict"] for flags in flag_sets])
     for v, flag_sets in testsuite.ALL_VARIANT_FLAGS.iteritems()
 )
 
 FAST_VARIANT_FLAGS_STRICT = dict(
@@ skipping 42 matching lines @@
 
 class Test262TestSuite(testsuite.TestSuite):
 
   def __init__(self, name, root):
     super(Test262TestSuite, self).__init__(name, root)
     self.testroot = os.path.join(self.root, *TEST_262_SUITE_PATH)
     self.harnesspath = os.path.join(self.root, *TEST_262_HARNESS_PATH)
     self.harness = [os.path.join(self.harnesspath, f)
                     for f in TEST_262_HARNESS_FILES]
     self.harness += [os.path.join(self.root, "harness-adapt.js")]
+    self.localtestroot = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
     self.ParseTestRecord = None
 
   def ListTests(self, context):
     tests = []
-    for dirname, dirs, files in os.walk(self.testroot):
+    for dirname, dirs, files in itertools.chain(os.walk(self.testroot),
+                                                os.walk(self.localtestroot)):
       for dotted in [x for x in dirs if x.startswith(".")]:
         dirs.remove(dotted)
       if context.noi18n and "intl402" in dirs:
         dirs.remove("intl402")
       dirs.sort()
       files.sort()
       for filename in files:
-        if filename.endswith(".js") and not filename.endswith("_FIXTURE.js"):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.testroot) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          case = testcase.TestCase(self, testname)
-          tests.append(case)
+        if not filename.endswith(".js"):
+          continue
+        if filename.endswith("_FIXTURE.js"):
+          continue
+        fullpath = os.path.join(dirname, filename)
+        # Match the (...) in '/path/to/v8/test/test262/subdir/test/(...).js'

[Michael Achenbach, 2017/01/05 14:06:32]

Just to clarify: subdir is one of data,local-tests

[Dan Ehrenberg, 2017/01/05 16:24:16]

Right, added this in a comment.

+        relpath = re.match(
+            r'.*[\\/]test[\\/]test262[\\/][^\\/]+[\\/]test[\\/](.*)\.js',

[Michael Achenbach, 2017/01/05 14:06:32]

Maybe keep the regexp in a toplevel constant for readability.

[Dan Ehrenberg, 2017/01/05 16:24:16]

Done.

+            fullpath).group(1)
+        # Defer to local-tests if there is a matching test of the same name
+        if dirname.startswith(self.testroot) and \

[Michael Achenbach, 2017/01/05 14:06:32]

nit: Rather use () around the whole condition instead of \

[Dan Ehrenberg, 2017/01/05 16:24:16]

This code no longer exists.

+            os.path.exists(os.path.join(self.localtestroot, relpath+'.js')):

[Michael Achenbach, 2017/01/05 14:06:32]

nit: spaces around +

or how about adding the .js to the group above? Wouldn't the regexp work like
this:
test[\\/](.*\.js)'

of course then you need the code relpath[:-3] below again, like on the left
side... as you wish.

[Michael Achenbach, 2017/01/05 14:06:32]

Do these "exists" checks give a measurable slowdown? E.g. on windows? We now
have this extra os call for every test file.

If there is any issue with that, we could also walk the tests in localtestroot
separately and then simply compare in memory which ones we found. I assume the
number of tests in localtestroot is significantly smaller than the tests in
data.

[Dan Ehrenberg, 2017/01/05 16:24:16]

This doesn't either.

[Dan Ehrenberg, 2017/01/05 16:24:16]

Turns out all we actually have to do is prune duplicates, so I did that using a
set instead. I don't have a great setup to test performance across platforms; if
you could let me know if you see any regressions, that'd be great.

[Michael Achenbach, 2017/01/06 09:26:02]

Yes, much nicer! The exists check is already encapsulated in the getpath method.
If we get any perf regression we could precalculate the existing local tests
once and then check those against each other test, but lets not go so far now.

+          continue
+        testname = relpath.replace(os.path.sep, "/")
+        case = testcase.TestCase(self, testname)
+        tests.append(case)
     return tests
 
   def GetFlagsForTestCase(self, testcase, context):
     return (testcase.flags + context.mode_flags + self.harness +
             self.GetIncludesForTest(testcase) + ["--harmony"] +
             (["--module"] if "module" in self.GetTestRecord(testcase) else []) +
-            [os.path.join(self.testroot, testcase.path + ".js")] +
+            [self.GetPathForTest(testcase)] +
             (["--throws"] if "negative" in self.GetTestRecord(testcase)
                           else []) +
             (["--allow-natives-syntax"]
              if "detachArrayBuffer.js" in
                 self.GetTestRecord(testcase).get("includes", [])
              else []))
 
   def _VariantGeneratorFactory(self):
     return Test262VariantGenerator
 
@@ skipping 25 matching lines @@
 
   def GetIncludesForTest(self, testcase):
     test_record = self.GetTestRecord(testcase)
     if "includes" in test_record:
       return [os.path.join(self.BasePath(filename), filename)
               for filename in test_record.get("includes", [])]
     else:
       includes = []
     return includes
 
+  def GetPathForTest(self, testcase):
+    filename = os.path.join(self.localtestroot, testcase.path + ".js")
+    if not os.path.exists(filename):
+      filename = os.path.join(self.testroot, testcase.path + ".js")
+    return filename
+
   def GetSourceForTest(self, testcase):
-    filename = os.path.join(self.testroot, testcase.path + ".js")

[Michael Achenbach, 2017/01/05 14:06:32]

This seemed to have worked because python seems tolerant. Strictly, the
testcase.path has only forward slashes also on windows. But I guess it's fine as
it worked before.

[Dan Ehrenberg, 2017/01/05 16:24:16]

I'll venture a guess that this is because Windows tolerates pathnames with / in
them, and we might be out of luck on Mac OS 9...

-    with open(filename) as f:
+    with open(self.GetPathForTest(testcase)) as f:
       return f.read()
 
   def _ParseException(self, str):
     # somefile:somelinenumber: someerror[: sometext]
     match = re.search('^[^: ]*:[0-9]+: ([^ ]+?)($|: )', str, re.MULTILINE)
     return match.group(1)
 
   def IsFailureOutput(self, testcase):
     output = testcase.output
     test_record = self.GetTestRecord(testcase)
@@ skipping 17 matching lines @@
     # data folder.
     if os.path.exists(ARCHIVE) and not os.path.exists(DATA):
       print "Extracting archive..."
       tar = tarfile.open(ARCHIVE)
       tar.extractall(path=os.path.dirname(ARCHIVE))
       tar.close()
 
 
 def GetSuite(name, root):
   return Test262TestSuite(name, root)