Directly use TestImporter in DepsUpdater instead of invoking script.

third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer.py

Index: third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer.py
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer.py b/third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer.py
index 8c2ae7f252d681c68ad162ce3a0399766b351652..b1afad069c1cb34ca00f86994290327572f42bf1 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer.py
@@ -25,125 +25,43 @@
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-"""This script imports a directory of W3C tests into Blink.
-
-This script takes a source repository directory, which it searches for files,
-then converts and copies files over to a destination directory.
-
-Rules for importing:
-
- * By default, only reference tests and JS tests are imported, (because pixel
-   tests take longer to run). This can be overridden with the --all flag.
-
- * By default, if test files by the same name already exist in the destination
-   directory, they are overwritten. This is because this script is used to
-   refresh files periodically. This can be overridden with the --no-overwrite flag.
-
- * All files are converted to work in Blink:
-     1. All CSS properties requiring the -webkit- vendor prefix are prefixed
-        (the list of what needs prefixes is read from Source/core/css/CSSProperties.in).
-     2. Each reftest has its own copy of its reference file following
-        the naming conventions new-run-webkit-tests expects.
-     3. If a reference files lives outside the directory of the test that
-        uses it, it is checked for paths to support files as it will be
-        imported into a different relative position to the test file
-        (in the same directory).
-     4. Any tags with the class "instructions" have style="display:none" added
-        to them. Some w3c tests contain instructions to manual testers which we
-        want to strip out (the test result parser only recognizes pure testharness.js
-        output and not those instructions).
-
- * Upon completion, script outputs the total number tests imported,
-   broken down by test type.
-
- * Also upon completion, if we are not importing the files in place, each
-   directory where files are imported will have a w3c-import.log file written with
-   a timestamp, the list of CSS properties used that require prefixes, the list
-   of imported files, and guidance for future test modification and maintenance.
-   On subsequent imports, this file is read to determine if files have been
-   removed in the newer changesets. The script removes these files accordingly.
+"""Logic for converting and copying files from a W3C repo.
+
+This module is responsible for modifying and copying a subset of the tests from
+a local W3C repository source directory into a destination directory.
 """
 
 import logging
 import mimetypes
-import re
-import optparse
 import os
-import sys
+import re
 
-from webkitpy.common.host import Host
 from webkitpy.common.webkit_finder import WebKitFinder
 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser
 from webkitpy.w3c.test_parser import TestParser
 from webkitpy.w3c.test_converter import convert_for_webkit
 
-
 # Maximum length of import path starting from top of source repository.
 # This limit is here because the Windows builders cannot create paths that are
 # longer than the Windows max path length (260). See http://crbug.com/609871.
 MAX_PATH_LENGTH = 125
 
-
 _log = logging.getLogger(__name__)
 
 
-def main(_argv, _stdout, _stderr):
-    options, args = parse_args()
-    host = Host()
-    source_repo_path = host.filesystem.normpath(os.path.abspath(args[0]))
-
-    if not host.filesystem.exists(source_repo_path):
-        sys.exit('Repository directory %s not found!' % source_repo_path)
-
-    configure_logging()
-    test_importer = TestImporter(host, source_repo_path, options)
-    test_importer.do_import()
-
-
-def configure_logging():
-    class LogHandler(logging.StreamHandler):
-
-        def format(self, record):
-            if record.levelno > logging.INFO:
-                return "%s: %s" % (record.levelname, record.getMessage())
-            return record.getMessage()
-
-    logger = logging.getLogger()
-    logger.setLevel(logging.INFO)
-    handler = LogHandler()
-    handler.setLevel(logging.INFO)
-    logger.addHandler(handler)
-    return handler
-
-
-def parse_args():
-    parser = optparse.OptionParser(usage='usage: %prog [options] source_repo_path')
-    parser.add_option('-n', '--no-overwrite', dest='overwrite', action='store_false', default=True,
-                      help=('Flag to prevent duplicate test files from overwriting existing tests. '
-                            'By default, they will be overwritten.'))
-    parser.add_option('-a', '--all', action='store_true', default=False,
-                      help=('Import all tests including reftests, JS tests, and manual/pixel tests. '
-                            'By default, only reftests and JS tests are imported.'))
-    parser.add_option('-d', '--dest-dir', dest='destination', default='w3c',
-                      help=('Import into a specified directory relative to the LayoutTests root. '
-                            'By default, files are imported under LayoutTests/w3c.'))
-    parser.add_option('--ignore-expectations', action='store_true', default=False,
-                      help='Ignore the W3CImportExpectations file and import everything.')
-    parser.add_option('--dry-run', action='store_true', default=False,
-                      help='Dryrun only (don\'t actually write any results).')
-
-    options, args = parser.parse_args()
-    if len(args) != 1:
-        parser.error('Incorrect number of arguments; source repo path is required.')
-    return options, args
-
-
 class TestImporter(object):
 
-    def __init__(self, host, source_repo_path, options):
+    def __init__(self, host, source_repo_path, dest_dir_name='imported'):
+        """Initializes variables to prepare for copying and converting files.
+
+        Args:
+            source_repo_path: Path to the local checkout of a WPT
+        """
         self.host = host
+
+        assert self.host.filesystem.exists(source_repo_path)
         self.source_repo_path = source_repo_path
-        self.options = options
+        self.dest_dir_name = dest_dir_name
 
         self.filesystem = self.host.filesystem
         self.webkit_finder = WebKitFinder(self.filesystem)
@@ -152,7 +70,7 @@ class TestImporter(object):
         self.destination_directory = self.filesystem.normpath(
             self.filesystem.join(
                 self.layout_tests_dir,
-                options.destination,
+                dest_dir_name,
                 self.filesystem.basename(self.source_repo_path)))
         self.import_in_place = (self.source_repo_path == self.destination_directory)
         self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
@@ -185,24 +103,24 @@ class TestImporter(object):
 
             # Files in 'tools' are not for browser testing, so we skip them.
             # See: http://testthewebforward.org/docs/test-format-guidelines.html#tools
-            DIRS_TO_SKIP = ('.git', 'test-plan', 'tools')
+            dirs_to_skip = ('.git', 'test-plan', 'tools')
 
             # We copy all files in 'support', including HTML without metadata.
             # See: http://testthewebforward.org/docs/test-format-guidelines.html#support-files
-            DIRS_TO_INCLUDE = ('resources', 'support')
+            dirs_to_include = ('resources', 'support')
 
             if dirs:
-                for d in DIRS_TO_SKIP:
-                    if d in dirs:
-                        dirs.remove(d)
+                for name in dirs_to_skip:

[jeffcarp, 2017/01/18 20:43:42]

It looks like these changes are also included in another CL - I'm assuming this
CL depends on the other one.

[qyearsley, 2017/01/18 23:10:58]

Right :-)

+                    if name in dirs:
+                        dirs.remove(name)
 
                 for path in paths_to_skip:
-                    path_base = path.replace(self.options.destination + '/', '')
+                    path_base = path.replace(self.dest_dir_name + '/', '')
                     path_base = path_base.replace(cur_dir, '')
                     path_full = self.filesystem.join(root, path_base)
                     if path_base in dirs:
                         dirs.remove(path_base)
-                        if not self.options.dry_run and self.import_in_place:
+                        if self.import_in_place:
                             _log.info("  pruning %s", path_base)
                             self.filesystem.rmtree(path_full)
                         else:
@@ -215,7 +133,7 @@ class TestImporter(object):
                 path_base = path_full.replace(self.source_repo_path + '/', '')
                 path_base = self.destination_directory.replace(self.layout_tests_dir + '/', '') + '/' + path_base
                 if path_base in paths_to_skip:
-                    if not self.options.dry_run and self.import_in_place:
+                    if self.import_in_place:
                         _log.info("  pruning %s", path_base)
                         self.filesystem.remove(path_full)
                         continue
@@ -240,7 +158,7 @@ class TestImporter(object):
                     copy_list.append({'src': fullpath, 'dest': filename})
                     continue
 
-                if self.filesystem.basename(root) in DIRS_TO_INCLUDE:
+                if self.filesystem.basename(root) in dirs_to_include:
                     copy_list.append({'src': fullpath, 'dest': filename})
                     continue
 
@@ -291,19 +209,12 @@ class TestImporter(object):
                     total_tests += 1
                     copy_list.append({'src': fullpath, 'dest': filename, 'is_jstest': True})
 
-                elif self.options.all:
-                    total_tests += 1
-                    copy_list.append({'src': fullpath, 'dest': filename})
-
             if copy_list:
                 # Only add this directory to the list if there's something to import
                 self.import_list.append({'dirname': root, 'copy_list': copy_list,
                                          'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests})
 
     def find_paths_to_skip(self):
-        if self.options.ignore_expectations:
-            return set()
-
         paths_to_skip = set()
         port = self.host.port_factory.get()
         w3c_import_expectations_path = self.webkit_finder.path_from_webkit_base('LayoutTests', 'W3CImportExpectations')
@@ -387,23 +298,17 @@ class TestImporter(object):
             _log.error('%s not found. Possible error in the test.', source_path)
             return None
 
-        if file_to_copy.get('reference_support_info'):
-            reference_support_info = file_to_copy['reference_support_info']
-        else:
-            reference_support_info = None
+        reference_support_info = file_to_copy.get('reference_support_info') or None
 
         if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
-            if not self.import_in_place and not self.options.dry_run:
+            if not self.import_in_place:
                 self.filesystem.maybe_make_directory(self.filesystem.dirname(dest_path))
 
         relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
-        if not self.options.overwrite and self.filesystem.exists(dest_path):
-            _log.info('  skipping %s', relpath)
-        else:
-            # FIXME: Maybe doing a file diff is in order here for existing files?
-            # In other words, there's no sense in overwriting identical files, but
-            # there's no harm in copying the identical thing.
-            _log.info('  %s', relpath)
+        # FIXME: Maybe doing a file diff is in order here for existing files?
+        # In other words, there's no sense in overwriting identical files, but
+        # there's no harm in copying the identical thing.
+        _log.info('  %s', relpath)
 
         if self.should_try_to_convert(file_to_copy, source_path, dest_dir):
             converted_file = convert_for_webkit(
@@ -414,10 +319,9 @@ class TestImporter(object):
                 self._prefixed_properties.setdefault(prefixed_property, 0)
                 self._prefixed_properties[prefixed_property] += 1
 
-            if not self.options.dry_run:
-                self.filesystem.write_text_file(dest_path, converted_file[1])
+            self.filesystem.write_text_file(dest_path, converted_file[1])
         else:
-            if not self.import_in_place and not self.options.dry_run:
+            if not self.import_in_place:
                 self.filesystem.copyfile(source_path, dest_path)
                 if self.filesystem.read_binary_file(source_path)[:2] == '#!':
                     self.filesystem.make_executable(dest_path)