Regenerate MANIFEST.json when WPT tests are run

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 18 matching lines @@
 """Abstract base class of Port-specific entry points for the layout tests
 test infrastructure (the Port and Driver classes).
 """
 
 import cgi
 import collections
 import difflib
 import errno
 import functools
 import json
+import hashlib
 import logging
 import optparse
 import re
 import sys
 
 from webkitpy.common import find_files
 from webkitpy.common import read_checksum_from_png
 from webkitpy.common.memoized import memoized
 from webkitpy.common.system.executive import ScriptError
 from webkitpy.common.system.path import cygpath, abspath_to_uri
@@ skipping 683 matching lines @@
             url = manifest_items[0][0]
             if url[1:] != path_in_wpt:
                 # TODO(tkent): foo.any.js and bar.worker.js should be accessed
                 # as foo.any.html, foo.any.worker, and bar.worker with WPTServe.
                 continue
             tests.append(file_path)
         return tests
 
     @memoized
     def _wpt_manifest(self):
-        path = self._filesystem.join(self.layout_tests_dir(), 'external', 'wpt', 'MANIFEST.json')
-        return json.loads(self._filesystem.read_text_file(path))
+        manifest_path = self._filesystem.join(self.layout_tests_dir(), 'external', 'wpt', 'MANIFEST.json')
+
+        original_hash = self._get_manifest_hash(manifest_path)
+        self._generate_manifest()
+        updated_hash = self._get_manifest_hash(manifest_path)
+
+        if original_hash != updated_hash:
+            _log.warning('MANIFEST.json has been updated to reflect changes in external/wpt!')

[jeffcarp, 2017/01/23 22:27:51]

I spent an hour trying to write a unit test for this and couldn't quite get
there. Replacing Port._generate_manifest caused weird errors like this:

  Traceback (most recent call last):
    File
"/usr/local/google/home/jeffcarp/chromium/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py",
line 316, in test_warn_if_new_test_found_in_manifest
      port.tests([])
    File
"/usr/local/google/home/jeffcarp/chromium/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py",
line 670, in tests
      tests = self.real_tests(paths)
    File
"/usr/local/google/home/jeffcarp/chromium/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py",
line 683, in real_tests
      skipped_directories, functools.partial(Port.is_test_file, self),
self.test_key)
    File
"/usr/local/google/home/jeffcarp/chromium/src/third_party/WebKit/Tools/Scripts/webkitpy/common/find_files.py",
line 63, in find
      filesystem, base_dir, paths), skipped_directories, file_filter,
directory_sort_key)
    File
"/usr/local/google/home/jeffcarp/chromium/src/third_party/WebKit/Tools/Scripts/webkitpy/common/find_files.py",
line 88, in _normalized_find
      for path in paths_to_walk))
  TypeError: type object argument after * must be a sequence, not generator

[jeffcarp, 2017/01/24 20:32:39]

Replacing line 269 of base_unittest.py below with this is enough to cause the
error:
port = self.make_port(with_tests=True,
executive=MockExecutive(run_command_fn=lambda: ''))

I need to mock out the call that _generate_manifest makes so that the
MANIFEST.json changes mid-run, which is why I was originally trying to replace
run_command_fn but it led to this issue. Currently looking into it.

[jeffcarp, 2017/01/24 20:42:59]

I figured it out (see newest patch). My guess is that if run_command_fn returned
anything, it caused the above code to break in that way. Returning nothing seems
to solve the problem.

+
+        return json.loads(self._filesystem.read_text_file(manifest_path))
+
+    def _get_manifest_hash(self, manifest_path):
+        hash_context = hashlib.md5()
+        manifest_buffer = self._filesystem.read_binary_file(manifest_path)
+        assert manifest_buffer
+        hash_context.update(manifest_buffer)
+        return hash_context.hexdigest()
+
+    def _generate_manifest(self):
+        wpt_path = self._webkit_finder.path_from_webkit_base('LayoutTests', 'external', 'wpt')
+        manifest_path = self._webkit_finder.path_from_webkit_base(
+            'Tools', 'Scripts', 'webkitpy', 'thirdparty', 'wpt', 'wpt', 'manifest')
+
+        _log.info('Generating MANIFEST.json')
+        self._executive.run_command([manifest_path, '--work', '--tests-root', wpt_path])
 
     def _manifest_items_for_path(self, path_in_wpt):
         """Returns a manifest item for the given WPT path, or None if not found.
 
         The format of a manifest item depends on
         https://github.com/w3c/wpt-tools/blob/master/manifest/item.py
         and is assumed to be a list of the format [url, extras],
         or [url, references, extras] for reftests, or None if not found.
 
         For most testharness tests, the returned manifest_items is expected
@@ skipping 889 matching lines @@
 
     def __init__(self, base, args, reference_args=None):
         self.name = base
         self.base = base
         self.args = args
         self.reference_args = args if reference_args is None else reference_args
         self.tests = set()
 
     def __repr__(self):
         return "PhysicalTestSuite('%s', '%s', %s, %s)" % (self.name, self.base, self.args, self.reference_args)