Run yapf on files in webkit/layout_tests/.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
 # in the documentation and/or other materials provided with the
 # distribution.
 #     * Neither the name of Google Inc. nor the names of its
 # contributors may be used to endorse or promote products derived from
 # this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 """
 The Manager runs a series of tests (TestType interface) against a set
 of test files.  If a test file fails a TestType, it returns a list of TestFailure
 objects to the Manager. The Manager then aggregates the TestFailures to
 create a final report.
 """
 
 import datetime
 import json
 import logging
@@ skipping 13 matching lines @@
 from webkitpy.tool import grammar
 
 _log = logging.getLogger(__name__)
 
 # Builder base URL where we have the archived test results.
 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
 
 TestExpectations = test_expectations.TestExpectations
 
 
-
 class Manager(object):
     """A class for managing running a series of tests on a series of layout
     test files."""
 
     def __init__(self, port, options, printer):
         """Initialize test runner data structures.
 
         Args:
           port: an object implementing port-specific
           options: a dictionary of command line options
           printer: a Printer object to record updates to.
         """
         self._port = port
         self._filesystem = port.host.filesystem
         self._options = options
         self._printer = printer
         self._expectations = None
 
         self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR
         self.INSPECTOR_SUBDIR = 'inspector' + port.TEST_PATH_SEPARATOR
         self.PERF_SUBDIR = 'perf'
         self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR
-        self.VIRTUAL_HTTP_SUBDIR = port.TEST_PATH_SEPARATOR.join([
-            'virtual', 'stable', 'http'])
+        self.VIRTUAL_HTTP_SUBDIR = port.TEST_PATH_SEPARATOR.join(['virtual', 'stable', 'http'])
         self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
         self.ARCHIVED_RESULTS_LIMIT = 25
         self._http_server_started = False
         self._wptserve_started = False
         self._websockets_server_started = False
 
         self._results_directory = self._port.results_directory()
         self._finder = LayoutTestFinder(self._port, self._options)
         self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow)
 
     def _collect_tests(self, args):
-        return self._finder.find_tests(args, test_list=self._options.test_list,
-            fastest_percentile=self._options.fastest)
+        return self._finder.find_tests(args, test_list=self._options.test_list, fastest_percentile=self._options.fastest)
 
     def _is_http_test(self, test):
-        return (
-            test.startswith(self.HTTP_SUBDIR) or
-            self._is_websocket_test(test) or
-            self.VIRTUAL_HTTP_SUBDIR in test
-        )
+        return (test.startswith(self.HTTP_SUBDIR) or self._is_websocket_test(test) or self.VIRTUAL_HTTP_SUBDIR in test)
 
     def _is_inspector_test(self, test):
         return self.INSPECTOR_SUBDIR in test
 
     def _is_websocket_test(self, test):
         if self._port.is_wpt_enabled() and self._port.is_wpt_test(test):
             return False
 
         return self.WEBSOCKET_SUBDIR in test
 
@@ skipping 11 matching lines @@
             return tests_to_run, tests_to_skip
 
         # Create a sorted list of test files so the subset chunk,
         # if used, contains alphabetically consecutive tests.
         if self._options.order == 'natural':
             tests_to_run.sort(key=self._port.test_key)
         elif self._options.order == 'random':
             random.shuffle(tests_to_run)
         elif self._options.order == 'random-seeded':
             rnd = random.Random()
-            rnd.seed(4) # http://xkcd.com/221/
+            rnd.seed(4)  # http://xkcd.com/221/
             rnd.shuffle(tests_to_run)
 
         tests_to_run, tests_in_other_chunks = self._finder.split_into_chunks(tests_to_run)
         self._expectations.add_extra_skipped_tests(tests_in_other_chunks)
         tests_to_skip.update(tests_in_other_chunks)
 
         return tests_to_run, tests_to_skip
 
     def _test_input_for_file(self, test_file):
-        return TestInput(test_file,
+        return TestInput(
+            test_file,
             self._options.slow_time_out_ms if self._test_is_slow(test_file) else self._options.time_out_ms,
             self._test_requires_lock(test_file),
             should_add_missing_baselines=(self._options.new_test_results and not self._test_is_expected_missing(test_file)))
 
     def _test_requires_lock(self, test_file):
         """Return True if the test needs to be locked when
         running multiple copies of NRWTs. Perf tests are locked
         because heavy load caused by running other tests in parallel
         might cause some of them to timeout."""
         return self._is_http_test(test_file) or self._is_perf_test(test_file)
 
     def _test_is_expected_missing(self, test_file):
         expectations = self._expectations.model().get_expectations(test_file)
         return test_expectations.MISSING in expectations or test_expectations.NEEDS_REBASELINE in expectations or test_expectations.NEEDS_MANUAL_REBASELINE in expectations
 
     def _test_is_slow(self, test_file):
         return test_expectations.SLOW in self._expectations.model().get_expectations(test_file)
 
     def needs_servers(self, test_names):
         return any(self._test_requires_lock(test_name) for test_name in test_names)
 
     def _rename_results_folder(self):
         try:
-            timestamp = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime(self._filesystem.mtime(self._filesystem.join(self._results_directory, "results.html"))))
+            timestamp = time.strftime(
+                "%Y-%m-%d-%H-%M-%S",
+                time.localtime(self._filesystem.mtime(self._filesystem.join(self._results_directory, "results.html"))))
         except (IOError, OSError), e:
             # It might be possible that results.html was not generated in previous run, because the test
             # run was interrupted even before testing started. In those cases, don't archive the folder.
             # Simply override the current folder contents with new results.
             import errno
             if e.errno == errno.EEXIST or e.errno == errno.ENOENT:
                 self._printer.write_update("No results.html file found in previous run, skipping it.")
             return None
         archived_name = ''.join((self._filesystem.basename(self._results_directory), "_", timestamp))
         archived_path = self._filesystem.join(self._filesystem.dirname(self._results_directory), archived_name)
@@ skipping 81 matching lines @@
             should_retry_failures = len(paths) < len(test_names)
         else:
             should_retry_failures = self._options.retry_failures
 
         enabled_pixel_tests_in_retry = False
         try:
             self._start_servers(tests_to_run)
 
             num_workers = self._port.num_workers(int(self._options.child_processes))
 
-            initial_results = self._run_tests(
-                tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations,
-                num_workers)
+            initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations,
+                                              num_workers)
 
             # Don't retry failures when interrupted by user or failures limit exception.
-            should_retry_failures = should_retry_failures and not (initial_results.interrupted or initial_results.keyboard_interrupted)
+            should_retry_failures = should_retry_failures and not (initial_results.interrupted or
+                                                                   initial_results.keyboard_interrupted)
 
             tests_to_retry = self._tests_to_retry(initial_results)
             all_retry_results = []
             if should_retry_failures and tests_to_retry:
                 enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed()
 
                 for retry_attempt in xrange(1, self._options.num_retries + 1):
                     if not tests_to_retry:
                         break
 
                     _log.info('')
-                    _log.info('Retrying %s, attempt %d of %d...' %
-                              (grammar.pluralize('unexpected failure', len(tests_to_retry)),
-                               retry_attempt, self._options.num_retries))
+                    _log.info('Retrying %s, attempt %d of %d...' % (grammar.pluralize('unexpected failure', len(tests_to_retry)),
+                                                                    retry_attempt, self._options.num_retries))
 
                     retry_results = self._run_tests(tests_to_retry,
                                                     tests_to_skip=set(),
                                                     repeat_each=1,
                                                     iterations=1,
                                                     num_workers=num_workers,
                                                     retry_attempt=retry_attempt)
                     all_retry_results.append(retry_results)
 
                     tests_to_retry = self._tests_to_retry(retry_results)
 
                 if enabled_pixel_tests_in_retry:
                     self._options.pixel_tests = False
         finally:
             self._stop_servers()
             self._clean_up_run()
 
         # Some crash logs can take a long time to be written out so look
         # for new logs after the test run finishes.
         self._printer.write_update("looking for new crash logs")
         self._look_for_new_crash_logs(initial_results, start_time)
         for retry_attempt_results in all_retry_results:
             self._look_for_new_crash_logs(retry_attempt_results, start_time)
 
         _log.debug("summarizing results")
-        summarized_full_results = test_run_results.summarize_results(
-            self._port, self._expectations, initial_results, all_retry_results,
-            enabled_pixel_tests_in_retry)
-        summarized_failing_results = test_run_results.summarize_results(
-            self._port, self._expectations, initial_results, all_retry_results,
-            enabled_pixel_tests_in_retry, only_include_failing=True)
+        summarized_full_results = test_run_results.summarize_results(self._port, self._expectations, initial_results,
+                                                                     all_retry_results, enabled_pixel_tests_in_retry)
+        summarized_failing_results = test_run_results.summarize_results(self._port,
+                                                                        self._expectations,
+                                                                        initial_results,
+                                                                        all_retry_results,
+                                                                        enabled_pixel_tests_in_retry,
+                                                                        only_include_failing=True)
 
         exit_code = summarized_failing_results['num_regressions']
         if exit_code > test_run_results.MAX_FAILURES_EXIT_STATUS:
             _log.warning('num regressions (%d) exceeds max exit status (%d)' %
                          (exit_code, test_run_results.MAX_FAILURES_EXIT_STATUS))
             exit_code = test_run_results.MAX_FAILURES_EXIT_STATUS
 
         if not self._options.dry_run:
             self._write_json_files(summarized_full_results, summarized_failing_results, initial_results, running_all_tests)
 
             if self._options.write_full_results_to:
-                self._filesystem.copyfile(self._filesystem.join(self._results_directory, "full_results.json"),
-                                          self._options.write_full_results_to)
+                self._filesystem.copyfile(
+                    self._filesystem.join(self._results_directory, "full_results.json"), self._options.write_full_results_to)
 
             self._upload_json_files()
 
             results_path = self._filesystem.join(self._results_directory, "results.html")
             self._copy_results_html_file(results_path)
             if initial_results.keyboard_interrupted:
                 exit_code = test_run_results.INTERRUPTED_EXIT_STATUS
             else:
                 if initial_results.interrupted:
                     exit_code = test_run_results.EARLY_EXIT_STATUS
-                if self._options.show_results and (exit_code or (self._options.full_results_html and initial_results.total_failures)):
+                if self._options.show_results and (exit_code or
+                                                   (self._options.full_results_html and initial_results.total_failures)):
                     self._port.show_results_html_file(results_path)
                 self._printer.print_results(time.time() - start_time, initial_results, summarized_failing_results)
 
         self._check_for_stale_w3c_dir()
 
-        return test_run_results.RunDetails(
-            exit_code, summarized_full_results, summarized_failing_results,
-            initial_results, all_retry_results, enabled_pixel_tests_in_retry)
+        return test_run_results.RunDetails(exit_code, summarized_full_results, summarized_failing_results, initial_results,
+                                           all_retry_results, enabled_pixel_tests_in_retry)
 
-    def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations,
-                   num_workers, retry_attempt=0):
+    def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, num_workers, retry_attempt=0):
 
         test_inputs = []
         for _ in xrange(iterations):
             for test in tests_to_run:
                 for _ in xrange(repeat_each):
                     test_inputs.append(self._test_input_for_file(test))
-        return self._runner.run_tests(self._expectations, test_inputs,
-                                      tests_to_skip, num_workers, retry_attempt)
+        return self._runner.run_tests(self._expectations, test_inputs, tests_to_skip, num_workers, retry_attempt)
 
     def _start_servers(self, tests_to_run):
         if self._port.is_wpt_enabled() and any(self._port.is_wpt_test(test) for test in tests_to_run):
             self._printer.write_update('Starting WPTServe ...')
             self._port.start_wptserve()
             self._wptserve_started = True
 
-        if self._port.requires_http_server() or any((self._is_http_test(test) or self._is_inspector_test(test)) for test in tests_to_run):
+        if self._port.requires_http_server() or any((self._is_http_test(test) or self._is_inspector_test(test))
+                                                    for test in tests_to_run):
             self._printer.write_update('Starting HTTP server ...')
             self._port.start_http_server(additional_dirs={}, number_of_drivers=self._options.max_locked_shards)
             self._http_server_started = True
 
         if any(self._is_websocket_test(test) for test in tests_to_run):
             self._printer.write_update('Starting WebSocket server ...')
             self._port.start_websocket_server()
             self._websockets_server_started = True
 
     def _stop_servers(self):
@@ skipping 82 matching lines @@
             if self._filesystem.isdir(file_path) and self._results_directory in file_path:
                 results_directories.append(file_path)
         self._delete_dirs(results_directories)
 
         # Port specific clean-up.
         self._port.clobber_old_port_specific_results()
 
     def _tests_to_retry(self, run_results):
         # TODO(ojan): This should also check that result.type != test_expectations.MISSING since retrying missing expectations is silly.
         # But that's a bit tricky since we only consider the last retry attempt for the count of unexpected regressions.
-        return [result.test_name for result in run_results.unexpected_results_by_name.values() if result.type != test_expectations.PASS]
+        return [result.test_name
+                for result in run_results.unexpected_results_by_name.values() if result.type != test_expectations.PASS]
 
     def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests):
         _log.debug("Writing JSON files in %s." % self._results_directory)
 
         # FIXME: Upload stats.json to the server and delete times_ms.
         times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
         times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
         json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
 
         # Save out the times data so we can use it for --fastest in the future.
@@ skipping 17 matching lines @@
 
     def _upload_json_files(self):
         if not self._options.test_results_server:
             return
 
         if not self._options.master_name:
             _log.error("--test-results-server was set, but --master-name was not.  Not uploading JSON files.")
             return
 
         _log.debug("Uploading JSON files for builder: %s", self._options.builder_name)
-        attrs = [("builder", self._options.builder_name),
-                 ("testtype", self._options.step_name),
+        attrs = [("builder", self._options.builder_name), ("testtype", self._options.step_name),
                  ("master", self._options.master_name)]
 
-        files = [(file, self._filesystem.join(self._results_directory, file)) for file in ["failing_results.json", "full_results.json", "times_ms.json"]]
+        files = [(file, self._filesystem.join(self._results_directory, file))
+                 for file in ["failing_results.json", "full_results.json", "times_ms.json"]]
 
         url = "http://%s/testfile/upload" % self._options.test_results_server
         # Set uploading timeout in case appengine server is having problems.
         # 120 seconds are more than enough to upload test results.
         uploader = FileUploader(url, 120)
         try:
             response = uploader.upload_as_multipart_form_data(self._filesystem, files, attrs)
             if response:
                 if response.code == 200:
                     _log.debug("JSON uploaded.")
@@ skipping 12 matching lines @@
         if self._filesystem.exists(results_file):
             self._filesystem.copyfile(results_file, destination_path)
 
     def _stats_trie(self, initial_results):
         def _worker_number(worker_name):
             return int(worker_name.split('/')[1]) if worker_name else -1
 
         stats = {}
         for result in initial_results.results_by_name.values():
             if result.type != test_expectations.SKIP:
-                stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000))}
+                stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(
+                    result.test_run_time * 1000), int(result.total_run_time * 1000))}
         stats_trie = {}
         for name, value in stats.iteritems():
             json_results_generator.add_path_to_trie(name, value, stats_trie)
         return stats_trie