Modify w3c import script to be able to run over a whole repo.

Tools/Scripts/webkitpy/w3c/test_importer.py

 #!/usr/bin/env python
 
 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #
 # 1. Redistributions of source code must retain the above
 #    copyright notice, this list of conditions and the following
@@ skipping 92 matching lines @@
 
 TEST_STATUS_UNKNOWN = 'unknown'
 TEST_STATUS_APPROVED = 'approved'
 TEST_STATUS_SUBMITTED = 'submitted'
 
 CHANGESET_NOT_AVAILABLE = 'Not Available'
 
 
 def main(_argv, _stdout, _stderr):
     options, args = parse_args()
-    import_dir = validate_import_directory(args[0])
-    test_importer = TestImporter(Host(), import_dir, options)
+    import_dir = args[0]
+    if len(args) == 1:
+        repo_dir = os.path.dirname(import_dir)
+    else:
+        repo_dir = args[1]
+
+    if not os.path.exists(import_dir):
+        sys.exit('Source directory %s not found!' % import_dir)
+
+    if not os.path.exists(repo_dir):
+        sys.exit('Repository directory %s not found!' % repo_dir)
+    if not repo_dir in import_dir:
+        sys.exit('Repository directory %s must be a parent of %s' % (repo_dir, import_dir))
+
+    test_importer = TestImporter(Host(), import_dir, repo_dir, options)
     test_importer.do_import()
 
 
 def parse_args():
-    parser = optparse.OptionParser(usage='usage: %prog [options] w3c_test_directory')
+    parser = optparse.OptionParser(usage='usage: %prog [options] w3c_test_directory [repo_directory]')
     parser.add_option('-n', '--no-overwrite', dest='overwrite', action='store_false', default=True,
         help='Flag to prevent duplicate test files from overwriting existing tests. By default, they will be overwritten')
     parser.add_option('-a', '--all', action='store_true', default=False,
         help='Import all tests including reftests, JS tests, and manual/pixel tests. By default, only reftests and JS tests are imported')
 
     options, args = parser.parse_args()
-    if len(args) != 1:
+    if len(args) not in (1, 2):
         parser.error('Incorrect number of arguments')
     return options, args
 
 
-def validate_import_directory(import_dir):
-    if not os.path.exists(import_dir):
-        sys.exit('Source directory %s not found!' % import_dir)
-
-    # Make sure the tests are officially submitted to the W3C, either approved or
-    # submitted following their directory naming conventions
-    if import_dir.find('approved') == -1 and import_dir.find('submitted') == -1:
-        # If not pointed directly to the approved directory or to any submitted
-        # directory, check for a submitted subdirectory and go with that
-        import_dir = os.path.join(import_dir, 'submitted')
-        if not os.path.exists(os.path.join(import_dir)):
-            sys.exit('Unable to import tests that aren\'t approved or submitted to the W3C')
-
-    return import_dir
-
-
 class TestImporter(object):
 
-    def __init__(self, host, source_directory, options):
+    def __init__(self, host, source_directory, repo_dir, options):
         self.host = host
         self.source_directory = source_directory
         self.options = options
 
         self.filesystem = self.host.filesystem
+
         self._webkit_root = __file__.split(self.filesystem.sep + 'Tools')[0]
+        self.repo_dir = repo_dir
+        subdirs = os.path.dirname(os.path.relpath(source_directory, repo_dir))
 
-        self.destination_directory = self.path_from_webkit_root("LayoutTests", "csswg")
+        self.destination_directory = os.path.join(self.path_from_webkit_root("LayoutTests"), 'w3c', subdirs)
 
         self.changeset = CHANGESET_NOT_AVAILABLE
         self.test_status = TEST_STATUS_UNKNOWN
 
         self.import_list = []
 
     def path_from_webkit_root(self, *comps):
         return self.filesystem.abspath(self.filesystem.join(self._webkit_root, *comps))
 
     def do_import(self):
@@ skipping 10 matching lines @@
 
     def find_importable_tests(self, directory):
         # FIXME: use filesystem
         for root, dirs, files in os.walk(directory):
             print 'Scanning ' + root + '...'
             total_tests = 0
             reftests = 0
             jstests = 0
 
             # "archive" and "data" dirs are internal csswg things that live in every approved directory.
-            DIRS_TO_SKIP = ('.git', '.hg', 'data', 'archive')
+            # FIXME: skip 'incoming' tests for now, but we should rework the 'test_status' concept and
+            # support reading them as well.
+            DIRS_TO_SKIP = ('.git', '.hg', 'data', 'archive', 'incoming')
             for d in DIRS_TO_SKIP:
                 if d in dirs:
                     dirs.remove(d)
 
             copy_list = []
 
             for filename in files:
                 # FIXME: This block should really be a separate function, but the early-continues make that difficult.
 
                 if filename.startswith('.') or filename.endswith('.pl'):
@@ skipping 51 matching lines @@
                 # We can skip the support directory if no tests were found.
                 if 'support' in dirs:
                     dirs.remove('support')
 
             if copy_list:
                 # Only add this directory to the list if there's something to import
                 self.import_list.append({'dirname': root, 'copy_list': copy_list,
                     'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests})
 
     def import_tests(self):
-        if self.import_list:
-            self.setup_destination_directory()
-
         converter = W3CTestConverter()
         total_imported_tests = 0
         total_imported_reftests = 0
         total_imported_jstests = 0
         total_prefixed_properties = {}
 
         for dir_to_copy in self.import_list:
-
             total_imported_tests += dir_to_copy['total_tests']
             total_imported_reftests += dir_to_copy['reftests']
             total_imported_jstests += dir_to_copy['jstests']
 
             prefixed_properties = []
 
             if not dir_to_copy['copy_list']:
                 continue
 
-            # Build the subpath starting with the approved/submitted directory
             orig_path = dir_to_copy['dirname']
-            start = orig_path.find(self.test_status)
-            new_subpath = orig_path[start:len(orig_path)]
 
-            # Append the new subpath to the destination_directory
-            new_path = os.path.join(self.destination_directory, new_subpath)
+            subpath = os.path.relpath(orig_path, self.repo_dir)
+            new_path = os.path.join(self.destination_directory, subpath)
 
-            # Create the destination subdirectories if not there
             if not(os.path.exists(new_path)):
                 os.makedirs(new_path)
 
             copied_files = []
 
             for file_to_copy in dir_to_copy['copy_list']:
                 # FIXME: Split this block into a separate function.
                 orig_filepath = os.path.normpath(file_to_copy['src'])
 
-                assert(not os.path.isdir(orig_filepath))
+                if os.path.isdir(orig_filepath):
+                    # FIXME: Figure out what is triggering this and what to do about it.
+                    print 'Error: %s refers to a directory' % orig_filepath
+                    continue
 
                 if not(os.path.exists(orig_filepath)):
                     print 'Warning: ' + orig_filepath + ' not found. Possible error in the test.'
                     continue
 
                 new_filepath = os.path.join(new_path, file_to_copy['dest'])
 
                 if not(os.path.exists(os.path.dirname(new_filepath))):
                     os.makedirs(os.path.dirname(new_filepath))
 
@@ skipping 41 matching lines @@
         print "Properties needing prefixes (by count):"
         for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
             print "  %s: %s" % (prefixed_property, total_prefixed_properties[prefixed_property])
 
     def setup_destination_directory(self):
         """ Creates a destination directory that mirrors that of the source approved or submitted directory """
 
         self.update_test_status()
 
         start = self.source_directory.find(self.test_status)
-        new_subpath = self.source_directory[start:len(self.source_directory)]
+        new_subpath = self.source_directory[len(self.repo_dir):]
 
         destination_directory = os.path.join(self.destination_directory, new_subpath)
 
         if not os.path.exists(destination_directory):
             os.makedirs(destination_directory)
 
         print 'Tests will be imported into: ' + destination_directory
 
     def update_test_status(self):
         """ Sets the test status to either 'approved' or 'submitted' """
@@ skipping 52 matching lines @@
             for prop in prop_list:
                 import_log.write(prop + '\n')
         else:
             import_log.write('None\n')
         import_log.write('------------------------------------------------------------------------\n')
         import_log.write('List of files:\n')
         for item in file_list:
             import_log.write(item + '\n')
 
         import_log.close()