Regenerate MANIFEST.json when WPT tests are run

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 26 matching lines @@
 The Manager object has a constructor and one main method called run.
 """
 
 import json
 import logging
 import random
 import sys
 import time
 
 from webkitpy.common.net.file_uploader import FileUploader
+from webkitpy.common.webkit_finder import WebKitFinder
 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
 from webkitpy.layout_tests.layout_package import json_results_generator
 from webkitpy.layout_tests.models import test_expectations
 from webkitpy.layout_tests.models import test_failures
 from webkitpy.layout_tests.models import test_run_results
 from webkitpy.layout_tests.models.test_input import TestInput
 from webkitpy.tool import grammar
 
 _log = logging.getLogger(__name__)
 
 TestExpectations = test_expectations.TestExpectations
 
 
 class Manager(object):
     """A class for managing running a series of layout tests."""
 
     def __init__(self, port, options, printer):
         """Initialize test runner data structures.
 
         Args:
           port: An object implementing platform-specific functionality.
           options: An options argument which contains command line options.
           printer: A Printer object to record updates to.
         """
         self._port = port
+        self._executive = port.host.executive
         self._filesystem = port.host.filesystem
         self._options = options
         self._printer = printer
         self._expectations = None
 
         self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR
         self.INSPECTOR_SUBDIR = 'inspector' + port.TEST_PATH_SEPARATOR
         self.PERF_SUBDIR = 'perf'
         self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR
         self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
         self.ARCHIVED_RESULTS_LIMIT = 25
         self._http_server_started = False
         self._wptserve_started = False
         self._websockets_server_started = False
 
         self._results_directory = self._port.results_directory()
         self._finder = LayoutTestFinder(self._port, self._options)
+        self._webkit_finder = WebKitFinder(port.host.filesystem)
         self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow)
 
     def run(self, args):
         """Run the tests and return a RunDetails object with the results."""
         start_time = time.time()
         self._printer.write_update("Collecting tests ...")
         running_all_tests = False
+
+        # Regenerate MANIFEST.json from template, necessary for WPT metadata

[qyearsley, 2017/02/28 18:54:45]

Nit: period at end of comment. Could also spell out web-platform-tests,
optionally.

[jeffcarp, 2017/02/28 21:57:39]

Done

+        self._ensure_manifest()
+
         try:
             paths, all_test_names, running_all_tests = self._collect_tests(args)
         except IOError:
             # This is raised if --test-list doesn't exist
             return test_run_results.RunDetails(exit_code=test_run_results.NO_TESTS_EXIT_STATUS)
 
         # Create a sorted list of test files so the subset chunk,
         # if used, contains alphabetically consecutive tests.
         if self._options.order == 'natural':
             all_test_names.sort(key=self._port.test_key)
@@ skipping 430 matching lines @@
 
         stats = {}
         for result in initial_results.results_by_name.values():
             if result.type != test_expectations.SKIP:
                 stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(
                     result.test_run_time * 1000), int(result.total_run_time * 1000))}
         stats_trie = {}
         for name, value in stats.iteritems():
             json_results_generator.add_path_to_trie(name, value, stats_trie)
         return stats_trie
+
+    def _ensure_manifest(self):

[qyearsley, 2017/02/28 18:54:45]

Do you think it makes sense to make this a public static method or top-level
function in webkitpy/w3c/wpt_manifest.py?

[jeffcarp, 2017/02/28 21:57:39]

My one reservation with that is that this method is very layout tests-specific.
But the manifest-related logic should be all in one place, so wpt_manifest.py
makes sense. Could this be in a follow-up CL?

[qyearsley, 2017/02/28 22:04:31]

Yep, follow-up CL would be fine, definitely.

+        manifest_path = self._webkit_finder.path_from_webkit_base(
+            'LayoutTests', 'external', 'wpt', 'MANIFEST.json')

[qyearsley, 2017/02/28 18:54:45]

Optional idea: These paths relative to webkit base could potentially be
constants in webkitpy/w3c/wpt_manifest.py.

+        manifest_base_path = self._webkit_finder.path_from_webkit_base(
+            'LayoutTests', 'external', 'WPT_BASE_MANIFEST.json')

[qyearsley, 2017/02/28 18:54:44]

Here we're assuming that WPT_BASE_MANIFEST.json always exists.
This is fine, although you could also make this explicit:

    assert self._filesystem.exists(manifest_base_path)

Or, you could even make it just log an error and generate the whole manifest
from scratch anyway.

[jeffcarp, 2017/02/28 21:57:39]

I think it's OK to assume it always exists because it will be checked into
source control. To make sure that is always the case, I could write some sort of
sanity check or PRESUBMIT to make sure it's there. But this is the same
assumption we're making now on master with respect to MANIFEST.json.

[qyearsley, 2017/02/28 22:04:31]

Alright; optionally you could still add an assert

+
+        if not self._filesystem.exists(manifest_path):
+            self._filesystem.copyfile(manifest_base_path, manifest_path)
+
+        manifest_tool_path = self._webkit_finder.path_from_webkit_base(
+            'Tools', 'Scripts', 'webkitpy', 'thirdparty', 'wpt', 'wpt', 'manifest')
+        wpt_path = self._webkit_finder.path_from_webkit_base('LayoutTests', 'external', 'wpt')
+
+        self._printer.write_update('Generating MANIFEST.json for web-platform-tests ...')
+        self._executive.run_command(['python', manifest_tool_path, '--work', '--tests-root', wpt_path])