Checkpoint work to import new tests from the w3c repos.

Tools/Scripts/webkitpy/w3c/test_importer.py

-#!/usr/bin/env python
-
 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #
 # 1. Redistributions of source code must retain the above
 #    copyright notice, this list of conditions and the following
 #    disclaimer.
 # 2. Redistributions in binary form must reproduce the above
@@ skipping 176 matching lines @@
             self.changeset = self.host.executive.run_command(['hg', 'tip']).split('changeset:')[1]
         except (OSError, ScriptError):
             self.changeset = CHANGESET_NOT_AVAILABLE
 
     def find_importable_tests(self, directory):
         # FIXME: use filesystem
         paths_to_skip = self.find_paths_to_skip()
 
         for root, dirs, files in os.walk(directory):
             cur_dir = root.replace(self.layout_tests_dir + '/', '') + '/'
-            _log.info('Scanning ' + cur_dir + '...')
+            _log.info('  scanning ' + cur_dir + '...')
             total_tests = 0
             reftests = 0
             jstests = 0
 
             # "archive" and "data" dirs are internal csswg things that live in every approved directory.
             # FIXME: skip 'incoming' tests for now, but we should rework the 'test_status' concept and
             # support reading them as well.
             DIRS_TO_SKIP = ('.git', '.hg', 'data', 'archive', 'incoming')
             if dirs:
                 for d in DIRS_TO_SKIP:
                     if d in dirs:
                         dirs.remove(d)
 
                 for path in paths_to_skip:
                     path_base = path.replace(cur_dir, '')
                     path_full = self.filesystem.join(root, path_base)
                     if path_base in dirs:
                         dirs.remove(path_base)
                         if not self.options.dry_run and self.import_in_place:
-                            _log.info("Pruning %s" % path_full)
+                            _log.info("  pruning %s" % path_base)
                             self.filesystem.rmtree(path_full)
 
             copy_list = []
 
             for filename in files:
                 path_full = self.filesystem.join(root, filename)
                 path_base = path_full.replace(self.layout_tests_dir + '/', '')
                 if path_base in paths_to_skip:
                     if not self.options.dry_run and self.import_in_place:
-                        _log.info("Pruning %s" % path_base)
+                        _log.info("  pruning %s" % path_base)
                         self.filesystem.remove(path_full)
                         continue
                 # FIXME: This block should really be a separate function, but the early-continues make that difficult.
 
                 if filename.startswith('.') or filename.endswith('.pl'):
                     continue  # For some reason the w3c repo contains random perl scripts we don't care about.
 
                 fullpath = os.path.join(root, filename)
 
                 mimetype = mimetypes.guess_type(fullpath)
@@ skipping 113 matching lines @@
                     _log.warning('%s not found. Possible error in the test.', orig_filepath)
                     continue
 
                 new_filepath = os.path.join(new_path, file_to_copy['dest'])
 
                 if not(os.path.exists(os.path.dirname(new_filepath))):
                     if not self.import_in_place and not self.options.dry_run:
                         os.makedirs(os.path.dirname(new_filepath))
 
                 if not self.options.overwrite and os.path.exists(new_filepath):
-                    _log.info('Skipping import of existing file ' + new_filepath)
+                    _log.info('  skipping import of existing file ' + new_filepath)
                 else:
                     # FIXME: Maybe doing a file diff is in order here for existing files?
                     # In other words, there's no sense in overwriting identical files, but
                     # there's no harm in copying the identical thing.
-                    _log.info('Importing: %s', orig_filepath)
-                    _log.info('       As: %s', new_filepath)
+                    _log.info('  importing %s', os.path.relpath(new_filepath, self.layout_tests_dir))
 
                 # Only html, xml, or css should be converted
                 # FIXME: Eventually, so should js when support is added for this type of conversion
                 mimetype = mimetypes.guess_type(orig_filepath)
                 if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0])  or 'css' in str(mimetype[0]):
                     converted_file = convert_for_webkit(new_path, filename=orig_filepath)
 
                     if not converted_file:
                         if not self.import_in_place and not self.options.dry_run:
                             shutil.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                     else:
                         for prefixed_property in converted_file[0]:
                             total_prefixed_properties.setdefault(prefixed_property, 0)
                             total_prefixed_properties[prefixed_property] += 1
 
                         prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
                         if not self.options.dry_run:
                             outfile = open(new_filepath, 'wb')
                             outfile.write(converted_file[1])
                             outfile.close()
                 else:
                     if not self.import_in_place and not self.options.dry_run:
                         shutil.copyfile(orig_filepath, new_filepath)
 
                 copied_files.append(new_filepath.replace(self._webkit_root, ''))
 
-            if not self.options.import_in_place and not self.options.dry_run:
+            if not self.import_in_place and not self.options.dry_run:
                 self.remove_deleted_files(new_path, copied_files)
                 self.write_import_log(new_path, copied_files, prefixed_properties)
 
+        _log.info('')
         _log.info('Import complete')
-
+        _log.info('')
         _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
         _log.info('Imported %d reftests', total_imported_reftests)
         _log.info('Imported %d JS tests', total_imported_jstests)
         _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
         _log.info('')
         _log.info('Properties needing prefixes (by count):')
         for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
             _log.info('  %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
 
     def setup_destination_directory(self):
@@ skipping 64 matching lines @@
             for prop in prop_list:
                 import_log.write(prop + '\n')
         else:
             import_log.write('None\n')
         import_log.write('------------------------------------------------------------------------\n')
         import_log.write('List of files:\n')
         for item in file_list:
             import_log.write(item + '\n')
 
         import_log.close()