WIP: blink/run-webkit-tests: Add support for gtest API.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 # Copyright (C) 2011 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
@@ skipping 14 matching lines @@
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import logging
 import optparse
 import os
 import sys
+import time
 import traceback
 
 from webkitpy.common.host import Host
 from webkitpy.common.system.executive import Executive
 from webkitpy.layout_tests.controllers.manager import Manager
 from webkitpy.layout_tests.models import test_run_results
 from webkitpy.layout_tests.port.factory import configuration_options, platform_options
 from webkitpy.layout_tests.views import buildbot_results
 from webkitpy.layout_tests.views import printing
 from webkitpy.layout_tests.generate_results_dashboard import DashBoardGenerator
@@ skipping 279 matching lines @@
                 default=0,
                 help="Set the maximum number of locked shards"),
             optparse.make_option(
                 "--nocheck-sys-deps",
                 action="store_true",
                 default=False,
                 help="Don't check the system dependencies (themes)"),
             optparse.make_option(
                 "--order",
                 action="store",
-                default="natural",
+                default=None,
                 help=("determine the order in which the test cases will be run. "
                       "'none' == use the order in which the tests were listed "
                       "either in arguments or test list, "
                       "'natural' == use the natural order (default), "
-                      "'random-seeded' == randomize the test order using a fixed seed, "
+                      "'random-seeded=SEED' == randomize the test order using "
+                      "a fixed seed, "
                       "'random' == randomize the test order.")),
             optparse.make_option(
                 "--profile",
                 action="store_true",
                 help="Output per-test profile information."),
             optparse.make_option(
                 "--profiler",
                 action="store",
                 help="Output per-test profile information, using the specified profiler."),
             optparse.make_option(
@@ skipping 100 matching lines @@
             optparse.make_option(
                 "--test-results-server",
                 default="",
                 help="If specified, upload results json files to this appengine server."),
             optparse.make_option(
                 "--write-full-results-to",
                 help=("If specified, copy full_results.json from the results dir to the "
                       "specified path.")),
         ]))
 
+    # Support gtest command line interface
+    # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#running-test-programs-advanced-options
+    option_group_definitions.append(
+        ("gtest compatibility API Options", [
+            # Listing Tests Names
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#listing-test-names
+            #
+            # --gtest_list_tests
+            # Format is;
+            #  TestCase1.
+            #    TestName1
+            #    TestName2
+            #  TestCase2.
+            #    TestName
+            # optparse.make_option(
+            #     "--gtest_list_tests",
+            #     default="",
+            #     help=""),
+
+            # Filtering
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#running-a-subset-of-the-tests
+            #
+            # --gtest_filter
+            # : - Separator for multiple patterns
+            # * - Match anything
+            # ? - Match single
+            # - - (Starts with) Exclude these matches
+            # optparse.make_option(
+            #     "--gtest_filter",
+            #     default="",
+            #     help=""),
+
+            # Disabled tests
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#temporarily-disabling-tests
+            #
+            # --gtest_also_run_disabled_tests
+            # $GTEST_ALSO_RUN_DISABLED_TESTS
+            #
+            # --skipped=ignore
+            # --skip-failing-tests
+            # --ignore-flaky-tests
+            # optparse.make_option(
+            #     "--gtest_also_run_disabled_tests",
+            #     default=os.environ.get("GTEST_ALSO_RUN_DISABLED_TESTS", "0"),
+            #     type=bool,
+            #     action="store_true",
+            #     help=""),
+
+            # Repeating
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#repeating-the-tests
+            #
+            # --gtest_repeat=TIMES
+            #
+            # ?? --repeat-each
+            # ?? --iterations
+            optparse.make_option(
+                "--gtest_repeat",
+                default="1",
+                type="int",
+                help=""),
+
+            # Shuffling
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#shuffling-the-tests
+            #
+            # --gtest_shuffle
+            # $GTEST_SHUFFLE
+            #   1 == enable shuffle
+            optparse.make_option(
+                "--gtest_shuffle",
+                default=os.environ.get("GTEST_SHUFFLE", "0"),
+                action="store_true",
+                help=""),
+
+            # --gtest_random_seed=SEED
+            # $GTEST_RANDOM_SEED
+            #   0 == "use current time"
+            #   1->99999 == Random seed
+            # --order=random-seeded
+            optparse.make_option(
+                "--gtest_random_seed",
+                default=os.environ.get("GTEST_RANDOM_SEED", "-1"),
+                type="int",
+                help=""),
+
+            # Sharding
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#distributing-test-functions-to-multiple-machines
+            # $GTEST_TOTAL_SHARDS
+            # $GTEST_SHARD_INDEX -- [0, GTEST_TOTAL_SHARDS - 1]
+            # $GTEST_SHARD_STATUS_FILE -- ???
+
+            # --run-part
+            # --run-chunk
+            optparse.make_option(
+                "--gtest_shard_index",
+                default=os.environ.get("GTEST_SHARD_INDEX", "-1"),
+                type="int",
+                help=""),
+            optparse.make_option(
+                "--gtest_total_shards",
+                default=os.environ.get("GTEST_TOTAL_SHARDS", "-1"),
+                type="int",
+                help=""),
+
+            # Output
+            ###################################################################
+            # GTEST_OUTPUT
+            # --gtest_output
+            optparse.make_option(
+                "--gtest_output",
+                default="",
+                help=""),
+        ]))
+
+    # Isolate command line interface
+    option_group_definitions.append(

[Dirk Pranke, 2016/06/21 22:05:24]

I don't necessarily think we should add all of these flags, and I don't know
that they're all needed for swarming. Some of them are maybe just different
names for existing flags, e.g., gtest_repeat, gtest_shard_index. 

Is your plan that swarming will invoke run-webkit-tests directly, rather than
using a wrapper script like we do for telemetry? Life might be easier if we just
had a script in //testing/buildbot that implemented exactly the interface we
needed for swarming, rather than trying to add more overlapping flags to a
command line that is already way too big and complicated.

If you were dead set on adding the flags here and not having a wrapper script,
I'd like to see us remove some of the existing flags that are redundant.

The way you've done the comments for these things is also quite unlike any of
the other comments in the code.

+        ("Swarming & Isolate compatibility API Options", [
+            # --write-full-results-to
+            optparse.make_option(
+                "--isolated-script-test-output",
+                default="",
+                help=""),
+        ]))
+
     option_parser = optparse.OptionParser()
 
     for group_name, group_options in option_group_definitions:
         option_group = optparse.OptionGroup(option_parser, group_name)
         option_group.add_options(group_options)
         option_parser.add_option_group(option_group)
 
     return option_parser.parse_args(args)
 
 
 def _set_up_derived_options(port, options, args):
     """Sets the options values that depend on other options values."""
+
     if options.batch_size is None:
         options.batch_size = port.default_batch_size()
 
     if not options.child_processes:
         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
                                                  str(port.default_child_processes()))
     if not options.max_locked_shards:
         options.max_locked_shards = int(os.environ.get("WEBKIT_TEST_MAX_LOCKED_SHARDS",
                                                        str(port.default_max_locked_shards())))
 
@@ skipping 47 matching lines @@
 
         if not options.test_list:
             options.test_list = []
         options.test_list.append(port.host.filesystem.join(port.layout_tests_dir(), 'SmokeTests'))
         if not options.skipped:
             options.skipped = 'always'
 
     if not options.skipped:
         options.skipped = 'default'
 
+    # gtest API to existing arguments
+    ######################################################################
+    # Listing Tests Names
+    # FIXME: Put stuff here
+    # Filtering
+    # FIXME: Put stuff here
+    # Disabled tests
+    # FIXME: Put stuff here
+
+    # Repeating
+    if options.gtest_repeat > 1:
+        options.iterations = options.gtest_repeat
+
+    # Shuffling
+    if options.gtest_shuffle == 1:
+        if options.order:
+            _log.fatal("Can't supply --gtest_shuffle and --order")
+
+        seed = options.gtest_random_seed
+        if seed in (-1, 0):
+            seed = int(time.time())
+
+        options.order = 'random-seeded=%s' % seed
+
+    if options.gtest_random_seed != -1 and not options.order.startswith('random-seeded'):
+        _log.fatal("Can't supply --gtest_random_seed supplied, but order not set to random-seeded")
+
+    if options.order is None:
+        options.order = 'natural'
+
+    # Sharding
+    if options.gtest_shard_index != -1:
+        if options.gtest_total_shards == -1:
+            _log.fatal('Require --gtest_total_shards if --gtest_shard_index is set.')
+
+        if options.run_part:
+            _log.fatal("Can't supply --gtest sharding options and --run_part!")
+
+        options.run_part = '%i:%i' % (options.gtest_shard_index + 1, options.gtest_total_shards)
+
+    # Output
+    # FIXME: Put stuff here
+
 
 def _run_tests(port, options, args, printer):
     _set_up_derived_options(port, options, args)
     manager = Manager(port, options, printer)
     printer.print_config(port.results_directory())
     return manager.run(args)
 
 
 def run(port, options, args, logging_stream, stdout):
     logger = logging.getLogger()
@@ skipping 27 matching lines @@
             _log.debug("\t%s" % process)
 
         return run_details
 
     finally:
         printer.cleanup()
 
 if __name__ == '__main__':
     exit_code = main(sys.argv[1:], sys.stdout, sys.stderr)
     sys.exit(exit_code)