Remove layout_tests_mover.py and obsolete local commit param.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_tests_mover.py

-# Copyright (C) 2013 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Moves a directory of LayoutTests.
-
-Given a path to a directory of LayoutTests, moves that directory, including all recursive children,
-to the specified destination path. Updates all references in tests and resources to reflect the new
-location. Also moves any corresponding platform-specific expected results and updates the test
-expectations to reflect the move.
-
-If the destination directory does not exist, it and any missing parent directories are created. If
-the destination directory already exists, the child members of the origin directory are added to the
-destination directory. If any of the child members clash with existing members of the destination
-directory, the move fails.
-
-Note that when new entries are added to the test expectations, no attempt is made to group or merge
-them with existing entries. This should be be done manually and with lint-test-expectations.
-"""
-
-import copy
-import logging
-import optparse
-import os
-import re
-import urlparse
-
-from webkitpy.common.checkout.scm.detection import SCMDetector
-from webkitpy.common.host import Host
-from webkitpy.common.system.executive import Executive
-from webkitpy.common.system.filesystem import FileSystem
-from webkitpy.layout_tests.port.base import Port
-from webkitpy.layout_tests.models.test_expectations import TestExpectations
-
-
-logging.basicConfig()
-_log = logging.getLogger(__name__)
-_log.setLevel(logging.INFO)
-
-PLATFORM_DIRECTORY = 'platform'
-
-class LayoutTestsMover(object):
-
-    def __init__(self, port=None):
-        self._port = port
-        if not self._port:
-            host = Host()
-            # Given that we use include_overrides=False and model_all_expectations=True when
-            # constructing the TestExpectations object, it doesn't matter which Port object we use.
-            self._port = host.port_factory.get()
-            self._port.host.initialize_scm()
-        self._filesystem = self._port.host.filesystem
-        self._scm = self._port.host.scm()
-        self._layout_tests_root = self._port.layout_tests_dir()
-
-    def _scm_path(self, *paths):
-        return self._filesystem.join('LayoutTests', *paths)
-
-    def _is_child_path(self, parent, possible_child):
-        normalized_parent = self._filesystem.normpath(parent)
-        normalized_child = self._filesystem.normpath(possible_child)
-        # We need to add a trailing separator to parent to avoid returning true for cases like
-        # parent='/foo/b', and possible_child='/foo/bar/baz'.
-        return normalized_parent == normalized_child or normalized_child.startswith(normalized_parent + self._filesystem.sep)
-
-    def _move_path(self, path, origin, destination):
-        if not self._is_child_path(origin, path):
-            return path
-        return self._filesystem.normpath(self._filesystem.join(destination, self._filesystem.relpath(path, origin)))
-
-    def _validate_input(self):
-        if not self._filesystem.isdir(self._absolute_origin):
-            raise Exception('Source path %s is not a directory' % self._origin)
-        if not self._is_child_path(self._layout_tests_root, self._absolute_origin):
-            raise Exception('Source path %s is not in LayoutTests directory' % self._origin)
-        if self._filesystem.isfile(self._absolute_destination):
-            raise Exception('Destination path %s is a file' % self._destination)
-        if not self._is_child_path(self._layout_tests_root, self._absolute_destination):
-            raise Exception('Destination path %s is not in LayoutTests directory' % self._destination)
-
-        # If destination is an existing directory, we move the children of origin into destination.
-        # However, if any of the children of origin would clash with existing children of
-        # destination, we fail.
-        # FIXME: Consider adding support for recursively moving into an existing directory.
-        if self._filesystem.isdir(self._absolute_destination):
-            for file_path in self._filesystem.listdir(self._absolute_origin):
-                if self._filesystem.exists(self._filesystem.join(self._absolute_destination, file_path)):
-                    raise Exception('Origin path %s clashes with existing destination path %s' %
-                            (self._filesystem.join(self._origin, file_path), self._filesystem.join(self._destination, file_path)))
-
-    def _get_expectations_for_test(self, model, test_path):
-        """Given a TestExpectationsModel object, finds all expectations that match the specified
-        test, specified as a relative path. Handles the fact that expectations may be keyed by
-        directory.
-        """
-        expectations = set()
-        if model.has_test(test_path):
-            expectations.add(model.get_expectation_line(test_path))
-        test_path = self._filesystem.dirname(test_path)
-        while not test_path == '':
-            # The model requires a trailing slash for directories.
-            test_path_for_model = test_path + '/'
-            if model.has_test(test_path_for_model):
-                expectations.add(model.get_expectation_line(test_path_for_model))
-            test_path = self._filesystem.dirname(test_path)
-        return expectations
-
-    def _get_expectations(self, model, path):
-        """Given a TestExpectationsModel object, finds all expectations for all tests under the
-        specified relative path.
-        """
-        expectations = set()
-        for test in self._filesystem.files_under(self._filesystem.join(self._layout_tests_root, path), dirs_to_skip=['script-tests', 'resources'],
-                                                 file_filter=Port.is_test_file):
-            expectations = expectations.union(self._get_expectations_for_test(model, self._filesystem.relpath(test, self._layout_tests_root)))
-        return expectations
-
-    @staticmethod
-    def _clone_expectation_line_for_path(expectation_line, path):
-        """Clones a TestExpectationLine object and updates the clone to apply to the specified
-        relative path.
-        """
-        clone = copy.copy(expectation_line)
-        clone.original_string = re.compile(expectation_line.name).sub(path, expectation_line.original_string)
-        clone.name = path
-        clone.path = path
-        # FIXME: Should we search existing expectations for matches, like in
-        # TestExpectationsParser._collect_matching_tests()?
-        clone.matching_tests = [path]
-        return clone
-
-    def _update_expectations(self):
-        """Updates all test expectations that are affected by the move.
-        """
-        _log.info('Updating expectations')
-        test_expectations = TestExpectations(self._port, include_overrides=False, model_all_expectations=True)
-
-        for expectation in self._get_expectations(test_expectations.model(), self._origin):
-            path = expectation.path
-            if self._is_child_path(self._origin, path):
-                # If the existing expectation is a child of the moved path, we simply replace it
-                # with an expectation for the updated path.
-                new_path = self._move_path(path, self._origin, self._destination)
-                _log.debug('Updating expectation for %s to %s' % (path, new_path))
-                test_expectations.remove_expectation_line(path)
-                test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path))
-            else:
-                # If the existing expectation is not a child of the moved path, we have to leave it
-                # in place. But we also add a new expectation for the destination path.
-                new_path = self._destination
-                _log.warning('Copying expectation for %s to %s. You should check that these expectations are still correct.' %
-                             (path, new_path))
-                test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path))
-
-        expectations_file = self._port.path_to_generic_test_expectations_file()
-        self._filesystem.write_text_file(expectations_file,
-                                         TestExpectations.list_to_string(test_expectations._expectations, reconstitute_only_these=[]))
-        self._scm.add(self._filesystem.relpath(expectations_file, self._scm.checkout_root))
-
-    def _find_references(self, input_files):
-        """Attempts to find all references to other files in the supplied list of files. Returns a
-        dictionary that maps from an absolute file path to an array of reference strings.
-        """
-        reference_regex = re.compile(r'(?:(?:src=|href=|importScripts\(|url\()(?:"([^"]+)"|\'([^\']+)\')|url\(([^\)\'"]+)\))')
-        references = {}
-        for input_file in input_files:
-            matches = reference_regex.findall(self._filesystem.read_binary_file(input_file))
-            if matches:
-                references[input_file] = [filter(None, match)[0] for match in matches]
-        return references
-
-    def _get_updated_reference(self, root, reference):
-        """For a reference <reference> in a directory <root>, determines the updated reference.
-        Returns the the updated reference, or None if no update is required.
-        """
-        # If the reference is an absolute path or url, it's safe.
-        if reference.startswith('/') or urlparse.urlparse(reference).scheme:
-            return None
-
-        # Both the root path and the target of the reference my be subject to the move, so there are
-        # four cases to consider. In the case where both or neither are subject to the move, the
-        # reference doesn't need updating.
-        #
-        # This is true even if the reference includes superfluous dot segments which mention a moved
-        # directory, as dot segments are collapsed during URL normalization. For example, if
-        # foo.html contains a reference 'bar/../script.js', this remains valid (though ugly) even if
-        # bar/ is moved to baz/, because the reference is always normalized to 'script.js'.
-        absolute_reference = self._filesystem.normpath(self._filesystem.join(root, reference))
-        if self._is_child_path(self._absolute_origin, root) == self._is_child_path(self._absolute_origin, absolute_reference):
-            return None;
-
-        new_root = self._move_path(root, self._absolute_origin, self._absolute_destination)
-        new_absolute_reference = self._move_path(absolute_reference, self._absolute_origin, self._absolute_destination)
-        return self._filesystem.relpath(new_absolute_reference, new_root)
-
-    def _get_all_updated_references(self, references):
-        """Determines the updated references due to the move. Returns a dictionary that maps from an
-        absolute file path to a dictionary that maps from a reference string to the corresponding
-        updated reference.
-        """
-        updates = {}
-        for file_path in references.keys():
-            root = self._filesystem.dirname(file_path)
-            # script-tests/TEMPLATE.html files contain references which are written as if the file
-            # were in the parent directory. This special-casing is ugly, but there are plans to
-            # remove script-tests.
-            if root.endswith('script-tests') and file_path.endswith('TEMPLATE.html'):
-                root = self._filesystem.dirname(root)
-            local_updates = {}
-            for reference in references[file_path]:
-                update = self._get_updated_reference(root, reference)
-                if update:
-                    local_updates[reference] = update
-            if local_updates:
-                updates[file_path] = local_updates
-        return updates
-
-    def _update_file(self, path, updates):
-        contents = self._filesystem.read_binary_file(path)
-        # Note that this regex isn't quite as strict as that used to find the references, but this
-        # avoids the need for alternative match groups, which simplifies things.
-        for target in updates.keys():
-            regex = re.compile(r'((?:src=|href=|importScripts\(|url\()["\']?)%s(["\']?)' % target)
-            contents = regex.sub(r'\1%s\2' % updates[target], contents)
-        self._filesystem.write_binary_file(path, contents)
-        self._scm.add(path)
-
-    def _update_test_source_files(self):
-        def is_test_source_file(filesystem, dirname, basename):
-            pass_regex = re.compile(r'\.(css|js)$')
-            fail_regex = re.compile(r'-expected\.')
-            return (Port.is_test_file(filesystem, dirname, basename) or pass_regex.search(basename)) and not fail_regex.search(basename)
-
-        test_source_files = self._filesystem.files_under(self._layout_tests_root, file_filter=is_test_source_file)
-        _log.info('Considering %s test source files for references' % len(test_source_files))
-        references = self._find_references(test_source_files)
-        _log.info('Considering references in %s files' % len(references))
-        updates = self._get_all_updated_references(references)
-        _log.info('Updating references in %s files' % len(updates))
-        count = 0
-        for file_path in updates.keys():
-            self._update_file(file_path, updates[file_path])
-            count += 1
-            if count % 1000 == 0 or count == len(updates):
-                _log.debug('Updated references in %s files' % count)
-
-    def _move_directory(self, origin, destination):
-        """Moves the directory <origin> to <destination>. If <destination> is a directory, moves the
-        children of <origin> into <destination>. Uses relative paths.
-        """
-        absolute_origin = self._filesystem.join(self._layout_tests_root, origin)
-        if not self._filesystem.isdir(absolute_origin):
-            return
-        _log.info('Moving directory %s to %s' % (origin, destination))
-        # Note that FileSystem.move() may silently overwrite existing files, but we
-        # check for this in _validate_input().
-        absolute_destination = self._filesystem.join(self._layout_tests_root, destination)
-        self._filesystem.maybe_make_directory(absolute_destination)
-        for directory in self._filesystem.listdir(absolute_origin):
-            self._scm.move(self._scm_path(origin, directory), self._scm_path(destination, directory))
-        self._filesystem.rmtree(absolute_origin)
-
-    def _move_files(self):
-        """Moves the all files that correspond to the move, including platform-specific expected
-        results.
-        """
-        self._move_directory(self._origin, self._destination)
-        for directory in self._filesystem.listdir(self._filesystem.join(self._layout_tests_root, PLATFORM_DIRECTORY)):
-            self._move_directory(self._filesystem.join(PLATFORM_DIRECTORY, directory, self._origin),
-                           self._filesystem.join(PLATFORM_DIRECTORY, directory, self._destination))
-
-    def _commit_changes(self):
-        if not self._scm.supports_local_commits():
-            return
-        title = 'Move LayoutTests directory %s to %s' % (self._origin, self._destination)
-        _log.info('Committing change \'%s\'' % title)
-        self._scm.commit_locally_with_message('%s\n\nThis commit was automatically generated by move-layout-tests.' % title,
-                                              commit_all_working_directory_changes=False)
-
-    def move(self, origin, destination):
-        self._origin = origin
-        self._destination = destination
-        self._absolute_origin = self._filesystem.join(self._layout_tests_root, self._origin)
-        self._absolute_destination = self._filesystem.join(self._layout_tests_root, self._destination)
-        self._validate_input()
-        self._update_expectations()
-        self._update_test_source_files()
-        self._move_files()
-        # FIXME: Handle virtual test suites.
-        self._commit_changes()
-
-def main(argv):
-    parser = optparse.OptionParser(description=__doc__)
-    parser.add_option('--origin',
-                      help=('The directory of tests to move, as a relative path from the LayoutTests directory.'))
-    parser.add_option('--destination',
-                      help=('The new path for the directory of tests, as a relative path from the LayoutTests directory.'))
-    options, _ = parser.parse_args()
-    LayoutTestsMover().move(options.origin, options.destination)