Support variable substitution and tryserver- and waterfall-only cmd line args.

scripts/slave/recipes/chromium_trybot.expected/dynamic_gtest_with_args_on_trybot.json

 [
   {
     "cmd": [
       "python",
       "-u",
       "[BUILD]/scripts/slave/bot_update.py",
       "--master",
       "tryserver.chromium.linux",
       "--builder",
       "linux_chromium_rel_ng",
@@ skipping 120 matching lines @@
       "/path/to/tmp/json"
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "read test spec",
     "~followup_annotations": [
       "@@@STEP_TEXT@path: [SLAVE_BUILD]/src/testing/buildbot/chromium.linux.json@@@",
       "@@@STEP_LOG_LINE@json.output@{@@@",
       "@@@STEP_LOG_LINE@json.output@  \"Linux Tests\": {@@@",
       "@@@STEP_LOG_LINE@json.output@    \"gtest_tests\": [@@@",
       "@@@STEP_LOG_LINE@json.output@      {@@@",
-      "@@@STEP_LOG_LINE@json.output@        \"swarming\": {@@@",
-      "@@@STEP_LOG_LINE@json.output@          \"can_use_on_swarming_builders\": true@@@",
-      "@@@STEP_LOG_LINE@json.output@        }, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"args\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@          \"--correct-common-arg\"@@@",
+      "@@@STEP_LOG_LINE@json.output@        ], @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"non_precommit_args\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@          \"--THIS-ARG-SHOULD-NOT-BE-PRESENT\"@@@",

[Sergiy Byelozyorov, 2016/01/14 09:43:47]

This is actually confusing when reading expectations. This is a test spec, so it
should be present here, but not below.

+      "@@@STEP_LOG_LINE@json.output@        ], @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"precommit_args\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@          \"--these-args-should-be-present\", @@@",
+      "@@@STEP_LOG_LINE@json.output@          \"--mastername=\\\"${master_name}\\\"\", @@@",
+      "@@@STEP_LOG_LINE@json.output@          \"--buildername=\\\"${builder_name}\\\"\", @@@",
+      "@@@STEP_LOG_LINE@json.output@          \"--buildrevision=\\\"${build_revision}\\\"\"@@@",
+      "@@@STEP_LOG_LINE@json.output@        ], @@@",
       "@@@STEP_LOG_LINE@json.output@        \"test\": \"base_unittests\"@@@",
       "@@@STEP_LOG_LINE@json.output@      }@@@",
       "@@@STEP_LOG_LINE@json.output@    ]@@@",
       "@@@STEP_LOG_LINE@json.output@  }@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {
     "cmd": [
@@ skipping 13 matching lines @@
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "get compile targets for scripts",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@json.output@{}@@@",
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {
     "cmd": [
-      "python",
-      "-u",
-      "RECIPE_MODULE[isolate]/resources/find_isolated_tests.py",
-      "--build-dir",
-      "[SLAVE_BUILD]/src/out/Release",
-      "--output-json",
-      "/path/to/tmp/json"
-    ],
-    "cwd": "[SLAVE_BUILD]",
-    "name": "find isolated tests",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@json.output@{@@@",
-      "@@@STEP_LOG_LINE@json.output@  \"dummy_target_1\": \"[dummy hash for dummy_target_1]\", @@@",
-      "@@@STEP_LOG_LINE@json.output@  \"dummy_target_2\": \"[dummy hash for dummy_target_2]\"@@@",
-      "@@@STEP_LOG_LINE@json.output@}@@@",
-      "@@@STEP_LOG_END@json.output@@@",
-      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"dummy_target_1\": \"[dummy hash for dummy_target_1]\", \"dummy_target_2\": \"[dummy hash for dummy_target_2]\"}@@@"
-    ]
-  },
-  {
-    "cmd": [
       "killall",
       "-9",
       "gnome-keyring-daemon"
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "killall gnome-keyring-daemon"
   },
   {
     "cmd": [
       "git",
@@ skipping 46 matching lines @@
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@excluded_files@foo.cc (regex = 'f.*')@@@",
       "@@@STEP_LOG_END@excluded_files@@@",
       "@@@STEP_LOG_LINE@analyze_details@{@@@",
       "@@@STEP_LOG_LINE@analyze_details@  \"additional_compile_targets\": [@@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"chromium_swarm_tests\"@@@",
       "@@@STEP_LOG_LINE@analyze_details@  ], @@@",
       "@@@STEP_LOG_LINE@analyze_details@  \"compile_targets\": [@@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"angle_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests\", @@@",
-      "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"chromium_swarm_tests\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"content_gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"tab_capture_end2end_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"telemetry_gpu_test_run\"@@@",
       "@@@STEP_LOG_LINE@analyze_details@  ], @@@",
       "@@@STEP_LOG_LINE@analyze_details@  \"self.m.filter.compile_targets\": [@@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"angle_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests\", @@@",
-      "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"chromium_swarm_tests\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"content_gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"tab_capture_end2end_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"telemetry_gpu_test_run\"@@@",
       "@@@STEP_LOG_LINE@analyze_details@  ], @@@",
       "@@@STEP_LOG_LINE@analyze_details@  \"self.m.filter.test_targets\": [@@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"angle_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests\", @@@",
-      "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"content_gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"tab_capture_end2end_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"telemetry_gpu_test_run\"@@@",
       "@@@STEP_LOG_LINE@analyze_details@  ], @@@",
       "@@@STEP_LOG_LINE@analyze_details@  \"test_targets\": [@@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"angle_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests\", @@@",
-      "@@@STEP_LOG_LINE@analyze_details@    \"base_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"content_gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"gl_unittests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"tab_capture_end2end_tests_run\", @@@",
       "@@@STEP_LOG_LINE@analyze_details@    \"telemetry_gpu_test_run\"@@@",
       "@@@STEP_LOG_LINE@analyze_details@  ]@@@",
       "@@@STEP_LOG_LINE@analyze_details@}@@@",
       "@@@STEP_LOG_END@analyze_details@@@"
     ]
   },
@@ skipping 26 matching lines @@
       "gen",
       "-m",
       "tryserver.chromium.linux",
       "-b",
       "linux_chromium_rel_ng",
       "--config-file",
       "[SLAVE_BUILD]/src/tools/mb/mb_config.pyl",
       "--goma-dir",
       "[BUILD]/goma",
       "--swarming-targets-file",
-      "base_unittests\ncontent_gl_tests\ngl_tests\nangle_unittests\ngl_unittests\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntab_capture_end2end_tests\nbase_unittests\n",
+      "content_gl_tests\ngl_tests\nangle_unittests\ngl_unittests\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntelemetry_gpu_test\ntab_capture_end2end_tests\n",
       "//out/Release"
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "generate_build_files (with patch)"
   },
   {
     "cmd": [
       "python",
       "-u",
       "[BUILD]/scripts/slave/compile.py",
       "--target",
       "Release",
       "--src-dir",
       "[SLAVE_BUILD]/src",
       "--build-tool",
       "ninja",
       "--compiler",
       "goma",
       "--goma-jsonstatus",
       "/path/to/tmp/json",
       "--goma-dir",
       "[BUILD]/goma",
       "--goma-fail-fast",
       "--goma-disable-local-fallback",
       "--ninja-ensure-up-to-date",
       "--",
       "angle_unittests_run",
       "base_unittests",
-      "base_unittests_run",
       "chromium_swarm_tests",
       "content_gl_tests_run",
       "gl_tests_run",
       "gl_unittests_run",
       "tab_capture_end2end_tests_run",
       "telemetry_gpu_test_run"
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "compile (with patch)",
     "~followup_annotations": [
@@ skipping 22 matching lines @@
       "[SLAVE_BUILD]/src/tools/swarming_client",
       "batcharchive",
       "--dump-json",
       "/path/to/tmp/json",
       "--isolate-server",
       "https://isolateserver.appspot.com",
       "--verbose",
       "[SLAVE_BUILD]/src/out/Release/content_gl_tests.isolated.gen.json",
       "[SLAVE_BUILD]/src/out/Release/gl_unittests.isolated.gen.json",
       "[SLAVE_BUILD]/src/out/Release/tab_capture_end2end_tests.isolated.gen.json",
-      "[SLAVE_BUILD]/src/out/Release/base_unittests.isolated.gen.json",
       "[SLAVE_BUILD]/src/out/Release/gl_tests.isolated.gen.json",
       "[SLAVE_BUILD]/src/out/Release/angle_unittests.isolated.gen.json",
       "[SLAVE_BUILD]/src/out/Release/telemetry_gpu_test.isolated.gen.json"
     ],
     "cwd": "[SLAVE_BUILD]",
     "name": "isolate tests",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@json.output@{@@@",
       "@@@STEP_LOG_LINE@json.output@  \"angle_unittests\": \"[dummy hash for angle_unittests]\", @@@",
-      "@@@STEP_LOG_LINE@json.output@  \"base_unittests\": \"[dummy hash for base_unittests]\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"content_gl_tests\": \"[dummy hash for content_gl_tests]\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"gl_tests\": \"[dummy hash for gl_tests]\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"gl_unittests\": \"[dummy hash for gl_unittests]\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"tab_capture_end2end_tests\": \"[dummy hash for tab_capture_end2end_tests]\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"telemetry_gpu_test\": \"[dummy hash for telemetry_gpu_test]\"@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {
     "cmd": [
       "python",
       "-u",
-      "\nimport sys\nprint '*.isolated file for target %s is missing' % sys.argv[1]\nsys.exit(1)\n",
-      "base_unittests"
-    ],
-    "cwd": "[SLAVE_BUILD]",
-    "name": "[error] base_unittests (with patch)",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@print '*.isolated file for target %s is missing' % sys.argv[1]@@@",
-      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
       "[SLAVE_BUILD]/src/tools/swarming_client/swarming.py",
       "trigger",
       "--swarming",
       "https://chromium-swarm.appspot.com",
       "--isolate-server",
       "https://isolateserver.appspot.com",
       "--priority",
       "30",
       "--shards",
       "1",
@@ skipping 1358 matching lines @@
       "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
       "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
       "@@@STEP_LOG_LINE@json.output@    }@@@",
       "@@@STEP_LOG_LINE@json.output@  }@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
       "@@@STEP_LOG_END@json.output@@@",
       "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@"
     ]
   },
   {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport sys\nprint '%s wasn\\'t triggered' % sys.argv[1]\nsys.exit(1)\n",
-      "base_unittests"
-    ],
-    "cwd": "[SLAVE_BUILD]",
-    "name": "[collect error] base_unittests (with patch)",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@print '%s wasn\\'t triggered' % sys.argv[1]@@@",
-      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
-  {
     "allow_subannotations": true,
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/src/infra/scripts/runtest_wrapper.py",
+      "--path-build",
+      "[BUILD]",
+      "--",
+      "--target",
+      "Release",
+      "--xvfb",
+      "--factory-properties",
+      "{\"blamelist\": [\"cool_dev1337@chromium.org\", \"hax@chromium.org\"], \"branch\": \"\", \"build_config\": \"Release\", \"buildername\": \"linux_chromium_rel_ng\", \"buildnumber\": 571, \"issue\": 12853011, \"mastername\": \"tryserver.chromium.linux\", \"patch_project\": \"chromium\", \"patchset\": 1, \"project\": \"chrome\", \"recipe\": \"chromium_trybot\", \"repository\": \"\", \"requester\": \"commit-bot@chromium.org\", \"revision\": null, \"rietveld\": \"https://codereview.chromium.org\", \"slavename\": \"TestSlavename\", \"swarm_hashes\": {\"angle_unittests\": \"[dummy hash for angle_unittests]\", \"content_gl_tests\": \"[dummy hash for content_gl_tests]\", \"gl_tests\": \"[dummy hash for gl_tests]\", \"gl_unittests\": \"[dummy hash for gl_unittests]\", \"gpu_unittests\": \"[dummy hash for gpu_unittests]\", \"tab_capture_end2end_tests\": \"[dummy hash for tab_capture_end2end_tests]\", \"telemetry_gpu_test\": \"[dummy hash for telemetry_gpu_test]\"}, \"workdir\": \"/path/to/workdir/TestSlavename\"}",
+      "--build-properties",
+      "{\"blamelist\": [\"cool_dev1337@chromium.org\", \"hax@chromium.org\"], \"branch\": \"\", \"build_config\": \"Release\", \"buildername\": \"linux_chromium_rel_ng\", \"buildnumber\": 571, \"issue\": 12853011, \"mastername\": \"tryserver.chromium.linux\", \"patch_project\": \"chromium\", \"patchset\": 1, \"project\": \"chrome\", \"recipe\": \"chromium_trybot\", \"repository\": \"\", \"requester\": \"commit-bot@chromium.org\", \"revision\": null, \"rietveld\": \"https://codereview.chromium.org\", \"slavename\": \"TestSlavename\", \"swarm_hashes\": {\"angle_unittests\": \"[dummy hash for angle_unittests]\", \"content_gl_tests\": \"[dummy hash for content_gl_tests]\", \"gl_tests\": \"[dummy hash for gl_tests]\", \"gl_unittests\": \"[dummy hash for gl_unittests]\", \"gpu_unittests\": \"[dummy hash for gpu_unittests]\", \"tab_capture_end2end_tests\": \"[dummy hash for tab_capture_end2end_tests]\", \"telemetry_gpu_test\": \"[dummy hash for telemetry_gpu_test]\"}, \"workdir\": \"/path/to/workdir/TestSlavename\"}",
+      "--annotate=gtest",
+      "--test-type=base_unittests",
+      "--step-name=base_unittests (with patch)",
+      "--test-launcher-summary-output",
+      "/path/to/tmp/json",
+      "--generate-json-file",
+      "-o",
+      "gtest-results/base_unittests",
+      "--builder-name=linux_chromium_rel_ng",
+      "--slave-name=TestSlavename",
+      "--build-number=571",
+      "base_unittests",
+      "--correct-common-arg",
+      "--these-args-should-be-present",
+      "--mastername=\"tryserver.chromium.linux\"",
+      "--buildername=\"linux_chromium_rel_ng\"",
+      "--buildrevision=\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\""
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "base_unittests (with patch)"
+  },
+  {
+    "allow_subannotations": true,
     "cmd": [
       "python",
       "-u",
       "RECIPE_MODULE[swarming]/resources/collect_gtest_task.py",
       "--swarming-client-dir",
       "[SLAVE_BUILD]/src/tools/swarming_client",
       "--temp-root-dir",
       "[TMP_BASE]",
       "--merged-test-output",
       "/path/to/tmp/json",
@@ skipping 476 matching lines @@
       "--build-number",
       "571",
       "--test-type",
       "tab_capture_end2end_tests on NVIDIA GPU on Linux (with patch) on Linux",
       "--test-results-server",
       "test-results.appspot.com",
       "--chrome-revision",
       "170242"
     ],
     "cwd": "[SLAVE_BUILD]",
-    "name": "Upload to test-results [tab_capture_end2end_tests on NVIDIA GPU on Linux (with patch) on Linux]",
-    "~followup_annotations": [
-      "@@@SET_BUILD_PROPERTY@failure_type@\"INVALID_TEST_RESULTS\"@@@"
-    ]
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "import sys; sys.exit(1)"
-    ],
-    "cwd": "[SLAVE_BUILD]",
-    "name": "base_unittests",
-    "~followup_annotations": [
-      "step returned non-zero exit code: 1",
-      "@@@STEP_TEXT@TEST RESULTS WERE INVALID@@@",
-      "@@@STEP_FAILURE@@@",
-      "@@@SET_BUILD_PROPERTY@failure_hash@\"212d0eb2385b1eaa828053689225258ece065725\"@@@"
-    ]
+    "name": "Upload to test-results [tab_capture_end2end_tests on NVIDIA GPU on Linux (with patch) on Linux]"
   },
   {
     "name": "$result",
-    "reason": "1 out of 1 aggregated steps failed. Failures: Step('base_unittests') failed with return_code 1",
-    "status_code": 1
+    "recipe_result": null,
+    "status_code": 0
   }
 ]