Revise update-w3c-deps to check in tests directly

Tools/Scripts/webkitpy/w3c/test_importer.py

 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #
 # 1. Redistributions of source code must retain the above
 #    copyright notice, this list of conditions and the following
 #    disclaimer.
 # 2. Redistributions in binary form must reproduce the above
@@ skipping 148 matching lines @@
         self.top_of_repo = top_of_repo
         self.options = options
 
         self.filesystem = self.host.filesystem
         self.webkit_finder = WebKitFinder(self.filesystem)
         self._webkit_root = self.webkit_finder.webkit_base()
         self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('LayoutTests')
         self.destination_directory = self.filesystem.normpath(self.filesystem.join(self.layout_tests_dir, options.destination,
                                                                                    self.filesystem.basename(self.top_of_repo)))
         self.import_in_place = (self.dir_to_import == self.destination_directory)
+        self.dir_above_repo = self.filesystem.dirname(self.top_of_repo)
 
         self.changeset = CHANGESET_NOT_AVAILABLE
 
         self.import_list = []
 
     def do_import(self):
         _log.info("Importing %s into %s", self.dir_to_import, self.destination_directory)
         self.find_importable_tests(self.dir_to_import)
         self.load_changeset()
         self.import_tests()
 
     def load_changeset(self):
         """Returns the current changeset from mercurial or "Not Available"."""
         try:
             self.changeset = self.host.executive.run_command(['hg', 'tip']).split('changeset:')[1]
         except (OSError, ScriptError):
             self.changeset = CHANGESET_NOT_AVAILABLE
 
     def find_importable_tests(self, directory):
         # FIXME: use filesystem
         paths_to_skip = self.find_paths_to_skip()
 
         for root, dirs, files in os.walk(directory):
-            cur_dir = root.replace(self.layout_tests_dir + '/', '') + '/'
+            cur_dir = root.replace(self.dir_above_repo + '/', '') + '/'
             _log.info('  scanning ' + cur_dir + '...')
             total_tests = 0
             reftests = 0
             jstests = 0
 
             DIRS_TO_SKIP = ('.git', '.hg')
             if dirs:
                 for d in DIRS_TO_SKIP:
                     if d in dirs:
                         dirs.remove(d)
 
                 for path in paths_to_skip:
-                    path_base = path.replace(cur_dir, '')
+                    path_base = path.replace(self.options.destination + '/', '')
+                    path_base = path_base.replace(cur_dir, '')
                     path_full = self.filesystem.join(root, path_base)
                     if path_base in dirs:
                         dirs.remove(path_base)
                         if not self.options.dry_run and self.import_in_place:
                             _log.info("  pruning %s" % path_base)
                             self.filesystem.rmtree(path_full)
+                        else:
+                            _log.info("  skipping %s" % path_base)
+
 
             copy_list = []
 
             for filename in files:
                 path_full = self.filesystem.join(root, filename)
                 path_base = path_full.replace(self.layout_tests_dir + '/', '')
                 if path_base in paths_to_skip:
                     if not self.options.dry_run and self.import_in_place:
                         _log.info("  pruning %s" % path_base)
                         self.filesystem.remove(path_full)
                         continue
+                    else:
+                        continue
                 # FIXME: This block should really be a separate function, but the early-continues make that difficult.
 
                 if filename.startswith('.') or filename.endswith('.pl'):
                     continue  # For some reason the w3c repo contains random perl scripts we don't care about.
 
                 fullpath = os.path.join(root, filename)
 
                 mimetype = mimetypes.guess_type(fullpath)
                 if not 'html' in str(mimetype[0]) and not 'application/xhtml+xml' in str(mimetype[0]) and not 'application/xml' in str(mimetype[0]):
                     copy_list.append({'src': fullpath, 'dest': filename})
@@ skipping 110 matching lines @@
                 if not(os.path.exists(orig_filepath)):
                     _log.warning('%s not found. Possible error in the test.', orig_filepath)
                     continue
 
                 new_filepath = os.path.join(new_path, file_to_copy['dest'])
 
                 if not(os.path.exists(os.path.dirname(new_filepath))):
                     if not self.import_in_place and not self.options.dry_run:
                         os.makedirs(os.path.dirname(new_filepath))
 
+                relpath = os.path.relpath(new_filepath, self.layout_tests_dir)
                 if not self.options.overwrite and os.path.exists(new_filepath):
-                    _log.info('  skipping import of existing file ' + new_filepath)
+                    _log.info('  skipping %s' % relpath)
                 else:
                     # FIXME: Maybe doing a file diff is in order here for existing files?
                     # In other words, there's no sense in overwriting identical files, but
                     # there's no harm in copying the identical thing.
-                    _log.info('  importing %s', os.path.relpath(new_filepath, self.layout_tests_dir))
+                    _log.info('  %s' % relpath)
 
                 # Only html, xml, or css should be converted
                 # FIXME: Eventually, so should js when support is added for this type of conversion
                 mimetype = mimetypes.guess_type(orig_filepath)
                 if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0])  or 'css' in str(mimetype[0]):
                     converted_file = convert_for_webkit(new_path, filename=orig_filepath)
 
                     if not converted_file:
                         if not self.import_in_place and not self.options.dry_run:
                             shutil.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                     else:
                         for prefixed_property in converted_file[0]:
                             total_prefixed_properties.setdefault(prefixed_property, 0)
                             total_prefixed_properties[prefixed_property] += 1
 
                         prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
                         if not self.options.dry_run:
                             outfile = open(new_filepath, 'wb')
                             outfile.write(converted_file[1])
                             outfile.close()
                 else:
                     if not self.import_in_place and not self.options.dry_run:
                         shutil.copyfile(orig_filepath, new_filepath)
 
                 copied_files.append(new_filepath.replace(self._webkit_root, ''))
 
-            if not self.import_in_place and not self.options.dry_run:
-                self.remove_deleted_files(new_path, copied_files)
-                self.write_import_log(new_path, copied_files, prefixed_properties)
-
         _log.info('')
         _log.info('Import complete')
         _log.info('')
         _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
         _log.info('Imported %d reftests', total_imported_reftests)
         _log.info('Imported %d JS tests', total_imported_jstests)
         _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
         _log.info('')
-        _log.info('Properties needing prefixes (by count):')
-        for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
-            _log.info('  %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
+
+        if total_prefixed_properties:
+            _log.info('Properties needing prefixes (by count):')
+            for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
+                _log.info('  %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
 
     def setup_destination_directory(self):
         """ Creates a destination directory that mirrors that of the source directory """
 
         new_subpath = self.dir_to_import[len(self.top_of_repo):]
 
         destination_directory = os.path.join(self.destination_directory, new_subpath)
 
         if not os.path.exists(destination_directory):
             os.makedirs(destination_directory)
 
         _log.info('Tests will be imported into: %s', destination_directory)
-
-    def remove_deleted_files(self, dir_to_import, new_file_list):
-        previous_file_list = []
-
-        import_log_file = os.path.join(dir_to_import, 'w3c-import.log')
-        if not os.path.exists(import_log_file):
-            return
-
-        import_log = open(import_log_file, 'r')
-        contents = import_log.readlines()
-
-        if 'List of files\n' in contents:
-            list_index = contents.index('List of files:\n') + 1
-            previous_file_list = [filename.strip() for filename in contents[list_index:]]
-
-        deleted_files = set(previous_file_list) - set(new_file_list)
-        for deleted_file in deleted_files:
-            _log.info('Deleting file removed from the W3C repo: %s', deleted_file)
-            deleted_file = os.path.join(self._webkit_root, deleted_file)
-            os.remove(deleted_file)
-
-        import_log.close()
-
-    def write_import_log(self, dir_to_import, file_list, prop_list):
-        now = datetime.datetime.now()
-
-        import_log = open(os.path.join(dir_to_import, 'w3c-import.log'), 'w')
-        import_log.write('The tests in this directory were imported from the W3C repository.\n')
-        import_log.write('Do NOT modify these tests directly in Webkit. Instead, push changes to the W3C CSS repo:\n\n')
-        import_log.write('http://hg.csswg.org/test\n\n')
-        import_log.write('Then run the Tools/Scripts/import-w3c-tests in Webkit to reimport\n\n')
-        import_log.write('Do NOT modify or remove this file\n\n')
-        import_log.write('------------------------------------------------------------------------\n')
-        import_log.write('Last Import: ' + now.strftime('%Y-%m-%d %H:%M') + '\n')
-        import_log.write('W3C Mercurial changeset: ' + self.changeset + '\n')
-        import_log.write('------------------------------------------------------------------------\n')
-        import_log.write('Properties requiring vendor prefixes:\n')
-        if prop_list:
-            for prop in prop_list:
-                import_log.write(prop + '\n')
-        else:
-            import_log.write('None\n')
-        import_log.write('------------------------------------------------------------------------\n')
-        import_log.write('List of files:\n')
-        for item in file_list:
-            import_log.write(item + '\n')
-
-        import_log.close()