WIP: blink/run-webkit-tests: Add support for gtest API.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

Index: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
index d01217095fafe55b7842be09c50d8b7091a519e0..f5c74a30558fdb86af208822c423a5687b9486ad 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
@@ -32,6 +32,7 @@ import logging
 import optparse
 import os
 import sys
+import time
 import traceback
 
 from webkitpy.common.host import Host
@@ -331,12 +332,13 @@ def parse_args(args):
             optparse.make_option(
                 "--order",
                 action="store",
-                default="natural",
+                default=None,
                 help=("determine the order in which the test cases will be run. "
                       "'none' == use the order in which the tests were listed "
                       "either in arguments or test list, "
                       "'natural' == use the natural order (default), "
-                      "'random-seeded' == randomize the test order using a fixed seed, "
+                      "'random-seeded=SEED' == randomize the test order using "
+                      "a fixed seed, "
                       "'random' == randomize the test order.")),
             optparse.make_option(
                 "--profile",
@@ -457,6 +459,135 @@ def parse_args(args):
                       "specified path.")),
         ]))
 
+    # Support gtest command line interface
+    # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#running-test-programs-advanced-options
+    option_group_definitions.append(
+        ("gtest compatibility API Options", [
+            # Listing Tests Names
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#listing-test-names
+            #
+            # --gtest_list_tests
+            # Format is;
+            #  TestCase1.
+            #    TestName1
+            #    TestName2
+            #  TestCase2.
+            #    TestName
+            # optparse.make_option(
+            #     "--gtest_list_tests",
+            #     default="",
+            #     help=""),
+
+            # Filtering
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#running-a-subset-of-the-tests
+            #
+            # --gtest_filter
+            # : - Separator for multiple patterns
+            # * - Match anything
+            # ? - Match single
+            # - - (Starts with) Exclude these matches
+            # optparse.make_option(
+            #     "--gtest_filter",
+            #     default="",
+            #     help=""),
+
+            # Disabled tests
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#temporarily-disabling-tests
+            #
+            # --gtest_also_run_disabled_tests
+            # $GTEST_ALSO_RUN_DISABLED_TESTS
+            #
+            # --skipped=ignore
+            # --skip-failing-tests
+            # --ignore-flaky-tests
+            # optparse.make_option(
+            #     "--gtest_also_run_disabled_tests",
+            #     default=os.environ.get("GTEST_ALSO_RUN_DISABLED_TESTS", "0"),
+            #     type=bool,
+            #     action="store_true",
+            #     help=""),
+
+            # Repeating
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#repeating-the-tests
+            #
+            # --gtest_repeat=TIMES
+            #
+            # ?? --repeat-each
+            # ?? --iterations
+            optparse.make_option(
+                "--gtest_repeat",
+                default="1",
+                type="int",
+                help=""),
+
+            # Shuffling
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#shuffling-the-tests
+            #
+            # --gtest_shuffle
+            # $GTEST_SHUFFLE
+            #   1 == enable shuffle
+            optparse.make_option(
+                "--gtest_shuffle",
+                default=os.environ.get("GTEST_SHUFFLE", "0"),
+                action="store_true",
+                help=""),
+
+            # --gtest_random_seed=SEED
+            # $GTEST_RANDOM_SEED
+            #   0 == "use current time"
+            #   1->99999 == Random seed
+            # --order=random-seeded
+            optparse.make_option(
+                "--gtest_random_seed",
+                default=os.environ.get("GTEST_RANDOM_SEED", "-1"),
+                type="int",
+                help=""),
+
+            # Sharding
+            ###################################################################
+            # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#distributing-test-functions-to-multiple-machines
+            # $GTEST_TOTAL_SHARDS
+            # $GTEST_SHARD_INDEX -- [0, GTEST_TOTAL_SHARDS - 1]
+            # $GTEST_SHARD_STATUS_FILE -- ???
+
+            # --run-part
+            # --run-chunk
+            optparse.make_option(
+                "--gtest_shard_index",
+                default=os.environ.get("GTEST_SHARD_INDEX", "-1"),
+                type="int",
+                help=""),
+            optparse.make_option(
+                "--gtest_total_shards",
+                default=os.environ.get("GTEST_TOTAL_SHARDS", "-1"),
+                type="int",
+                help=""),
+
+            # Output
+            ###################################################################
+            # GTEST_OUTPUT
+            # --gtest_output
+            optparse.make_option(
+                "--gtest_output",
+                default="",
+                help=""),
+        ]))
+
+    # Isolate command line interface
+    option_group_definitions.append(

[Dirk Pranke, 2016/06/21 22:05:24]

I don't necessarily think we should add all of these flags, and I don't know
that they're all needed for swarming. Some of them are maybe just different
names for existing flags, e.g., gtest_repeat, gtest_shard_index. 

Is your plan that swarming will invoke run-webkit-tests directly, rather than
using a wrapper script like we do for telemetry? Life might be easier if we just
had a script in //testing/buildbot that implemented exactly the interface we
needed for swarming, rather than trying to add more overlapping flags to a
command line that is already way too big and complicated.

If you were dead set on adding the flags here and not having a wrapper script,
I'd like to see us remove some of the existing flags that are redundant.

The way you've done the comments for these things is also quite unlike any of
the other comments in the code.

+        ("Swarming & Isolate compatibility API Options", [
+            # --write-full-results-to
+            optparse.make_option(
+                "--isolated-script-test-output",
+                default="",
+                help=""),
+        ]))
+
     option_parser = optparse.OptionParser()
 
     for group_name, group_options in option_group_definitions:
@@ -469,6 +600,7 @@ def parse_args(args):
 
 def _set_up_derived_options(port, options, args):
     """Sets the options values that depend on other options values."""
+
     if options.batch_size is None:
         options.batch_size = port.default_batch_size()
 
@@ -536,6 +668,49 @@ def _set_up_derived_options(port, options, args):
     if not options.skipped:
         options.skipped = 'default'
 
+    # gtest API to existing arguments
+    ######################################################################
+    # Listing Tests Names
+    # FIXME: Put stuff here
+    # Filtering
+    # FIXME: Put stuff here
+    # Disabled tests
+    # FIXME: Put stuff here
+
+    # Repeating
+    if options.gtest_repeat > 1:
+        options.iterations = options.gtest_repeat
+
+    # Shuffling
+    if options.gtest_shuffle == 1:
+        if options.order:
+            _log.fatal("Can't supply --gtest_shuffle and --order")
+
+        seed = options.gtest_random_seed
+        if seed in (-1, 0):
+            seed = int(time.time())
+
+        options.order = 'random-seeded=%s' % seed
+
+    if options.gtest_random_seed != -1 and not options.order.startswith('random-seeded'):
+        _log.fatal("Can't supply --gtest_random_seed supplied, but order not set to random-seeded")
+
+    if options.order is None:
+        options.order = 'natural'
+
+    # Sharding
+    if options.gtest_shard_index != -1:
+        if options.gtest_total_shards == -1:
+            _log.fatal('Require --gtest_total_shards if --gtest_shard_index is set.')
+
+        if options.run_part:
+            _log.fatal("Can't supply --gtest sharding options and --run_part!")
+
+        options.run_part = '%i:%i' % (options.gtest_shard_index + 1, options.gtest_total_shards)
+
+    # Output
+    # FIXME: Put stuff here
+
 
 def _run_tests(port, options, args, printer):
     _set_up_derived_options(port, options, args)