blink/run-webkit-tests: Allow seeding the random test run order and write out value.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 22 matching lines @@
 starting the required helper servers, deciding the order and way to
 run the tests, retrying fails tests and collecting the test results,
 including crash logs, and mismatches with expectations.
 
 The Manager object has a constructor and one main method called run.
 """
 
 import json
 import logging
 import random
+import re
 import sys
 import time
 
 from webkitpy.common.net.file_uploader import FileUploader
 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
 from webkitpy.layout_tests.layout_package import json_results_generator
 from webkitpy.layout_tests.models import test_expectations
 from webkitpy.layout_tests.models import test_failures
@@ skipping 29 matching lines @@
         self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR
         self.INSPECTOR_SUBDIR = 'inspector' + port.TEST_PATH_SEPARATOR
         self.PERF_SUBDIR = 'perf'
         self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR
         self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
         self.ARCHIVED_RESULTS_LIMIT = 25
         self._http_server_started = False
         self._wptserve_started = False
         self._websockets_server_started = False
 
+        self._random_seed = None
         self._results_directory = self._port.results_directory()
         self._finder = LayoutTestFinder(self._port, self._options)
         self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow)
 
     def run(self, args):
         """Run the tests and return a RunDetails object with the results."""
         start_time = time.time()
         self._printer.write_update("Collecting tests ...")
         running_all_tests = False
         try:
@@ skipping 10 matching lines @@
 
         # Check to make sure we're not skipping every test.
         if not tests_to_run:
             _log.critical('No tests to run.')
             return test_run_results.RunDetails(exit_code=test_run_results.NO_TESTS_EXIT_STATUS)
 
         exit_code = self._set_up_run(tests_to_run)
         if exit_code:
             return test_run_results.RunDetails(exit_code=exit_code)
 
+        # Write out the random seed and the order we are running the tests in.
+        if not self._options.dry_run:
+            if self._random_seed:
+                self._filesystem.write_text_file(
+                    self._filesystem.join(self._results_directory, "random-seed.txt"),
+                    "%s\n" % self._random_seed)
+            self._filesystem.write_text_file(
+                self._filesystem.join(self._results_directory, "tests-run-order.txt"),
+                "\n".join(tests_to_run))

[Dirk Pranke, 2016/06/21 23:47:13]

How does this relate to your "write a json file out" ?

See also the comment I had in that CL that I'd just log the seed instead ...

[mithro, 2016/06/22 08:40:08]

Logging the random seed into the json environment file would be an alternative
to logging to this file. I didn't want to block this patch on having the json
environment recording figured out and a simple txt file seemed the simplest
option.
I do log the random seed used on line 277. 

However, unless I'm missing something, the log output doesn't end up in the
results output? As we are generating a lot of layout test results with different
random seeds, it makes things a lot easier to have this information recorded in
the output result somewhere. Happy to put it somewhere else if you have a better
/ more appropriate idea.

The tests-run-order.txt was/is really useful when looking into these problems
but it is less required. I can remove this bit (or put it behind a flag?) it if
you prefer.

+
         # Don't retry failures if an explicit list of tests was passed in.
         if self._options.retry_failures is None:
             should_retry_failures = len(paths) < len(test_names)
         else:
             should_retry_failures = self._options.retry_failures
 
         enabled_pixel_tests_in_retry = False
         try:
             self._start_servers(tests_to_run)
 
@@ skipping 113 matching lines @@
         tests_to_skip = self._finder.skip_tests(paths, test_names, self._expectations, self._http_tests(test_names))
         tests_to_run = [test for test in test_names if test not in tests_to_skip]
 
         if not tests_to_run:
             return tests_to_run, tests_to_skip
 
         # Create a sorted list of test files so the subset chunk,
         # if used, contains alphabetically consecutive tests.
         if self._options.order == 'natural':
             tests_to_run.sort(key=self._port.test_key)
-        elif self._options.order == 'random':
-            random.shuffle(tests_to_run)
-        elif self._options.order == 'random-seeded':
+        elif self._options.order.startswith('random'):
             rnd = random.Random()

[qyearsley, 2016/06/22 17:30:16]

This line could be removed (it appears below).

-            rnd.seed(4)  # http://xkcd.com/221/
+
+            groups = re.match('random-seeded(=([0-9]+))?', self._options.order)

[qyearsley, 2016/06/22 17:30:16]

Instead of specifying seeds with an option like `--order=random-seeded=123`,
what do you think about adding a separate option "--seed" which takes an
integer? Then we also wouldn't need to have both --order=random and
--order=random-seeded; whether it's using an input seed or generating a new seed
could depend on whether --seed is passed in. For example:

    ...
    elif self._options.order == 'random':
        if self._options.seed is not None:
            seed = self._options.seed
        else:
            seed = int(time.time())
        rnd = random.Random()
        test_to_run.sort() 
        rnd.shuffle(tests_to_run)
        

[Dirk Pranke, 2016/06/24 01:53:40]

+1 to splitting it into a separate argument.

+            if not groups:
+                seed = int(time.time())
+            elif not groups.group(2):
+                seed = 4  # http://xkcd.com/221/

[qyearsley, 2016/06/22 17:30:16]

It's probably not necessary to necessary to keep this option here anymore; I
don't think anything depends on using --order=random-seeded.

+            else:
+                seed = int(groups.group(2))
+            _log.info("Test random order seed: %i", seed)
+            self._random_seed = seed
+
+            # Make sure the tests are in sorted before we randomize them.
+            tests_to_run.sort()
+            rnd = random.Random()
+            rnd.seed(seed)
             rnd.shuffle(tests_to_run)
 
         tests_to_run, tests_in_other_chunks = self._finder.split_into_chunks(tests_to_run)
         self._expectations.add_extra_skipped_tests(tests_in_other_chunks)
         tests_to_skip.update(tests_in_other_chunks)
 
         return tests_to_run, tests_to_skip
 
     def _test_input_for_file(self, test_file):
         return TestInput(test_file,
@@ skipping 283 matching lines @@
 
         stats = {}
         for result in initial_results.results_by_name.values():
             if result.type != test_expectations.SKIP:
                 stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(
                     result.test_run_time * 1000), int(result.total_run_time * 1000))}
         stats_trie = {}
         for name, value in stats.iteritems():
             json_results_generator.add_path_to_trie(name, value, stats_trie)
         return stats_trie