[test] Process to locally develop and upstream test262 tests

test/test262/testcfg.py

Index: test/test262/testcfg.py
diff --git a/test/test262/testcfg.py b/test/test262/testcfg.py
index d8123968fca0578e35855b0192e580944322656f..a22515e4c95ed786c6604211026303e806a398fd 100644
--- a/test/test262/testcfg.py
+++ b/test/test262/testcfg.py
@@ -27,6 +27,7 @@
 
 
 import imp
+import itertools
 import os
 import re
 import sys
@@ -47,6 +48,7 @@ TEST_262_NATIVE_FILES = ["detachArrayBuffer.js"]
 TEST_262_SUITE_PATH = ["data", "test"]
 TEST_262_HARNESS_PATH = ["data", "harness"]
 TEST_262_TOOLS_PATH = ["harness", "src"]
+TEST_262_LOCAL_TESTS_PATH = ["local-tests", "test"]
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                              *TEST_262_TOOLS_PATH))
@@ -109,11 +111,13 @@ class Test262TestSuite(testsuite.TestSuite):
     self.harness = [os.path.join(self.harnesspath, f)
                     for f in TEST_262_HARNESS_FILES]
     self.harness += [os.path.join(self.root, "harness-adapt.js")]
+    self.localtestroot = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
     self.ParseTestRecord = None
 
   def ListTests(self, context):
     tests = []
-    for dirname, dirs, files in os.walk(self.testroot):
+    for dirname, dirs, files in itertools.chain(os.walk(self.testroot),
+                                                os.walk(self.localtestroot)):
       for dotted in [x for x in dirs if x.startswith(".")]:
         dirs.remove(dotted)
       if context.noi18n and "intl402" in dirs:
@@ -121,19 +125,29 @@ class Test262TestSuite(testsuite.TestSuite):
       dirs.sort()
       files.sort()
       for filename in files:
-        if filename.endswith(".js") and not filename.endswith("_FIXTURE.js"):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.testroot) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          case = testcase.TestCase(self, testname)
-          tests.append(case)
+        if not filename.endswith(".js"):
+          continue
+        if filename.endswith("_FIXTURE.js"):
+          continue
+        fullpath = os.path.join(dirname, filename)
+        # Match the (...) in '/path/to/v8/test/test262/subdir/test/(...).js'

[Michael Achenbach, 2017/01/05 14:06:32]

Just to clarify: subdir is one of data,local-tests

[Dan Ehrenberg, 2017/01/05 16:24:16]

Right, added this in a comment.

+        relpath = re.match(
+            r'.*[\\/]test[\\/]test262[\\/][^\\/]+[\\/]test[\\/](.*)\.js',

[Michael Achenbach, 2017/01/05 14:06:32]

Maybe keep the regexp in a toplevel constant for readability.

[Dan Ehrenberg, 2017/01/05 16:24:16]

Done.

+            fullpath).group(1)
+        # Defer to local-tests if there is a matching test of the same name
+        if dirname.startswith(self.testroot) and \

[Michael Achenbach, 2017/01/05 14:06:32]

nit: Rather use () around the whole condition instead of \

[Dan Ehrenberg, 2017/01/05 16:24:16]

This code no longer exists.

+            os.path.exists(os.path.join(self.localtestroot, relpath+'.js')):

[Michael Achenbach, 2017/01/05 14:06:32]

nit: spaces around +

or how about adding the .js to the group above? Wouldn't the regexp work like
this:
test[\\/](.*\.js)'

of course then you need the code relpath[:-3] below again, like on the left
side... as you wish.

[Michael Achenbach, 2017/01/05 14:06:32]

Do these "exists" checks give a measurable slowdown? E.g. on windows? We now
have this extra os call for every test file.

If there is any issue with that, we could also walk the tests in localtestroot
separately and then simply compare in memory which ones we found. I assume the
number of tests in localtestroot is significantly smaller than the tests in
data.

[Dan Ehrenberg, 2017/01/05 16:24:16]

This doesn't either.

[Dan Ehrenberg, 2017/01/05 16:24:16]

Turns out all we actually have to do is prune duplicates, so I did that using a
set instead. I don't have a great setup to test performance across platforms; if
you could let me know if you see any regressions, that'd be great.

[Michael Achenbach, 2017/01/06 09:26:02]

Yes, much nicer! The exists check is already encapsulated in the getpath method.
If we get any perf regression we could precalculate the existing local tests
once and then check those against each other test, but lets not go so far now.

+          continue
+        testname = relpath.replace(os.path.sep, "/")
+        case = testcase.TestCase(self, testname)
+        tests.append(case)
     return tests
 
   def GetFlagsForTestCase(self, testcase, context):
     return (testcase.flags + context.mode_flags + self.harness +
             self.GetIncludesForTest(testcase) + ["--harmony"] +
             (["--module"] if "module" in self.GetTestRecord(testcase) else []) +
-            [os.path.join(self.testroot, testcase.path + ".js")] +
+            [self.GetPathForTest(testcase)] +
             (["--throws"] if "negative" in self.GetTestRecord(testcase)
                           else []) +
             (["--allow-natives-syntax"]
@@ -179,9 +193,14 @@ class Test262TestSuite(testsuite.TestSuite):
       includes = []
     return includes
 
+  def GetPathForTest(self, testcase):
+    filename = os.path.join(self.localtestroot, testcase.path + ".js")
+    if not os.path.exists(filename):
+      filename = os.path.join(self.testroot, testcase.path + ".js")
+    return filename
+
   def GetSourceForTest(self, testcase):
-    filename = os.path.join(self.testroot, testcase.path + ".js")

[Michael Achenbach, 2017/01/05 14:06:32]

This seemed to have worked because python seems tolerant. Strictly, the
testcase.path has only forward slashes also on windows. But I guess it's fine as
it worked before.

[Dan Ehrenberg, 2017/01/05 16:24:16]

I'll venture a guess that this is because Windows tolerates pathnames with / in
them, and we might be out of luck on Mac OS 9...

-    with open(filename) as f:
+    with open(self.GetPathForTest(testcase)) as f:
       return f.read()
 
   def _ParseException(self, str):