Add json test results format support for SwarmingIsolatedScriptTest

scripts/slave/recipes/chromium.expected/dynamic_swarmed_sharded_invalid_format_isolated_script_test.json

 [
   {
     "cmd": [
       "python",
       "-u",
       "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
       "--spec",
       "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}]",
       "--patch_root",
       "src",
@@ skipping 208 matching lines @@
     "~followup_annotations": [
       "@@@STEP_TEXT@path: [SLAVE_BUILD]/src/testing/buildbot/chromium.linux.json@@@",
       "@@@STEP_LOG_LINE@json.output@{@@@",
       "@@@STEP_LOG_LINE@json.output@  \"Linux Tests\": {@@@",
       "@@@STEP_LOG_LINE@json.output@    \"isolated_scripts\": [@@@",
       "@@@STEP_LOG_LINE@json.output@      {@@@",
       "@@@STEP_LOG_LINE@json.output@        \"isolate_name\": \"telemetry_gpu_unittests\", @@@",
       "@@@STEP_LOG_LINE@json.output@        \"name\": \"telemetry_gpu_unittests\", @@@",
       "@@@STEP_LOG_LINE@json.output@        \"swarming\": {@@@",
       "@@@STEP_LOG_LINE@json.output@          \"can_use_on_swarming_builders\": true, @@@",
-      "@@@STEP_LOG_LINE@json.output@          \"shards\": 2@@@",
+      "@@@STEP_LOG_LINE@json.output@          \"shards\": 3@@@",
       "@@@STEP_LOG_LINE@json.output@        }@@@",
       "@@@STEP_LOG_LINE@json.output@      }@@@",
       "@@@STEP_LOG_LINE@json.output@    ]@@@",
       "@@@STEP_LOG_LINE@json.output@  }@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {
     "cmd": [
@@ skipping 79 matching lines @@
       "-u",
       "[SLAVE_BUILD]/src/tools/swarming_client/swarming.py",
       "trigger",
       "--swarming",
       "https://chromium-swarm.appspot.com",
       "--isolate-server",
       "https://isolateserver.appspot.com",
       "--priority",
       "25",
       "--shards",
-      "2",
+      "3",
       "--task-name",
       "telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571",
       "--dump-json",
       "/path/to/tmp/json",
       "--expiration",
       "3600",
       "--io-timeout",
       "1200",
       "--hard-timeout",
       "3600",
@@ skipping 37 matching lines @@
       "--isolated-script-test-chartjson-output=${ISOLATED_OUTDIR}/chartjson-output.json"
     ],
     "env": {
       "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
     },
     "name": "[trigger] telemetry_gpu_unittests on Ubuntu-12.04",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@json.output@{@@@",
       "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
-      "@@@STEP_LOG_LINE@json.output@    \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:2:0\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:3:0\": {@@@",
       "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
       "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
       "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
       "@@@STEP_LOG_LINE@json.output@    }, @@@",
-      "@@@STEP_LOG_LINE@json.output@    \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:2:1\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:3:1\": {@@@",
       "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 1, @@@",
       "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10100\", @@@",
       "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10100\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:3:2\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 2, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10200\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10200\"@@@",
       "@@@STEP_LOG_LINE@json.output@    }@@@",
       "@@@STEP_LOG_LINE@json.output@  }@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
       "@@@STEP_LOG_END@json.output@@@",
       "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
-      "@@@STEP_LINK@shard #1@https://chromium-swarm.appspot.com/user/task/10100@@@"
+      "@@@STEP_LINK@shard #1@https://chromium-swarm.appspot.com/user/task/10100@@@",
+      "@@@STEP_LINK@shard #2@https://chromium-swarm.appspot.com/user/task/10200@@@"
     ]
   },
   {
     "cmd": [
       "python",
       "-u",
       "[SLAVE_BUILD]/src/tools/swarming_client/swarming.py",
       "collect",
       "--swarming",
       "https://chromium-swarm.appspot.com",
       "--decorate",
       "--print-status-updates",
       "--json",
-      "{\"base_task_name\": \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571\", \"tasks\": {\"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:2:0\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}, \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:2:1\": {\"shard_index\": 1, \"task_id\": \"10100\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10100\"}}}",
+      "{\"base_task_name\": \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571\", \"tasks\": {\"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:3:0\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}, \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:3:1\": {\"shard_index\": 1, \"task_id\": \"10100\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10100\"}, \"telemetry_gpu_unittests/Ubuntu-12.04/eeeeeeeeee/Linux Tests/571:3:2\": {\"shard_index\": 2, \"task_id\": \"10200\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10200\"}}}",
       "--task-output-dir",
       "/path/to/tmp/"
     ],
     "env": {
       "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
     },
     "name": "telemetry_gpu_unittests on Ubuntu-12.04",
     "~followup_annotations": [
-      "step returned non-zero exit code: 1",
       "@@@STEP_LOG_LINE@outdir_json@{@@@",
       "@@@STEP_LOG_LINE@outdir_json@  \"0/chartjson-output.json\": \"{\\\"dummy\\\": \\\"dummy0\\\", \\\"enabled\\\": true, \\\"charts\\\": {\\\"entry2\\\": \\\"chart2\\\", \\\"entry1\\\": \\\"chart1\\\"}}\", @@@",
-      "@@@STEP_LOG_LINE@outdir_json@  \"0/output.json\": \"{\\\"successes\\\": [\\\"test1.Test1\\\", \\\"test2.Test2\\\"], \\\"failures\\\": [], \\\"valid\\\": true, \\\"times\\\": {\\\"test2.Test2\\\": 0.1, \\\"test1.Test1\\\": 0.1}}\", @@@",
-      "@@@STEP_LOG_LINE@outdir_json@  \"1/chartjson-output.json\": \"\", @@@",
-      "@@@STEP_LOG_LINE@outdir_json@  \"1/output.json\": \"{\\\"successes\\\": [\\\"test3.Test3\\\", \\\"test4.Test4\\\"], \\\"failures\\\": [], \\\"valid\\\": true, \\\"times\\\": {\\\"test3.Test3\\\": 0.1, \\\"test4.Test4\\\": 0.1}}\", @@@",
+      "@@@STEP_LOG_LINE@outdir_json@  \"0/output.json\": \"{\\\"tests\\\": {\\\"test1\\\": {\\\"Test1\\\": {\\\"expected\\\": \\\"PASS\\\", \\\"actual\\\": \\\"PASS\\\"}, \\\"Test2\\\": {\\\"expected\\\": \\\"PASS TIMEOUT\\\", \\\"actual\\\": \\\"TIMEOUT\\\"}}, \\\"test_common\\\": {\\\"Test1\\\": {\\\"expected\\\": \\\"PASS\\\", \\\"actual\\\": \\\"FAIL FAIL PASS\\\"}}}, \\\"interrupted\\\": false, \\\"version\\\": 3, \\\"seconds_since_epoch\\\": 14000000, \\\"num_failures_by_type\\\": {\\\"FAIL\\\": 0, \\\"PASS\\\": 2}}\", @@@",
+      "@@@STEP_LOG_LINE@outdir_json@  \"1/chartjson-output.json\": \"{\\\"dummy\\\": \\\"dummy1\\\", \\\"enabled\\\": true, \\\"charts\\\": {\\\"entry3\\\": \\\"chart3\\\", \\\"entry4\\\": \\\"chart4\\\"}}\", @@@",
+      "@@@STEP_LOG_LINE@outdir_json@  \"1/output.json\": \"{\\\"tests\\\": {\\\"test3\\\": {\\\"Test3\\\": {\\\"expected\\\": \\\"PASS\\\", \\\"actual\\\": \\\"PASS\\\"}, \\\"Test4\\\": {\\\"expected\\\": \\\"PASS TIMEOUT\\\", \\\"actual\\\": \\\"TIMEOUT\\\"}}, \\\"test_common\\\": {\\\"Test3\\\": {\\\"expected\\\": \\\"PASS\\\", \\\"actual\\\": \\\"FAIL FAIL PASS\\\"}}}, \\\"interrupted\\\": false, \\\"version\\\": 3, \\\"seconds_since_epoch\\\": 14000001, \\\"num_failures_by_type\\\": {\\\"FAIL\\\": 0, \\\"PASS\\\": 2}}\", @@@",
       "@@@STEP_LOG_LINE@outdir_json@  \"2/chartjson-output.json\": \"{\\\"dummy\\\": \\\"dummy2\\\", \\\"enabled\\\": true, \\\"charts\\\": {\\\"entry6\\\": \\\"chart6\\\", \\\"entry5\\\": \\\"chart5\\\"}}\", @@@",
-      "@@@STEP_LOG_LINE@outdir_json@  \"2/output.json\": \"{\\\"successes\\\": [\\\"test5.Test5\\\", \\\"test6.Test6\\\"], \\\"failures\\\": [], \\\"valid\\\": true, \\\"times\\\": {\\\"test6.Test6\\\": 0.1, \\\"test5.Test5\\\": 0.1}}\", @@@",
-      "@@@STEP_LOG_LINE@outdir_json@  \"3/chartjson-output.json\": \"{\\\"dummy\\\": \\\"dummy3\\\", \\\"enabled\\\": true, \\\"charts\\\": {\\\"entry8\\\": \\\"chart8\\\", \\\"entry7\\\": \\\"chart7\\\"}}\", @@@",
-      "@@@STEP_LOG_LINE@outdir_json@  \"3/output.json\": \"{\\\"successes\\\": [\\\"test7.Test7\\\", \\\"test8.Test8\\\"], \\\"failures\\\": [], \\\"valid\\\": true, \\\"times\\\": {\\\"test7.Test7\\\": 0.1, \\\"test8.Test8\\\": 0.1}}\", @@@",
-      "@@@STEP_LOG_LINE@outdir_json@  \"summary.json\": \"{\\\"shards\\\": [{\\\"internal_failure\\\": false, \\\"failure\\\": false, \\\"exit_code\\\": \\\"0\\\"}, {\\\"internal_failure\\\": false, \\\"failure\\\": false, \\\"exit_code\\\": \\\"0\\\"}, {\\\"internal_failure\\\": false, \\\"failure\\\": false, \\\"exit_code\\\": \\\"0\\\"}, {\\\"internal_failure\\\": false, \\\"failure\\\": false, \\\"exit_code\\\": \\\"0\\\"}]}\"@@@",
+      "@@@STEP_LOG_LINE@outdir_json@  \"2/output.json\": \"{\\\"tests\\\": {\\\"test_common\\\": {\\\"Test5\\\": {\\\"expected\\\": \\\"PASS\\\", \\\"actual\\\": \\\"FAIL FAIL PASS\\\"}}, \\\"test5\\\": {\\\"Test5\\\": {\\\"expected\\\": \\\"PASS\\\", \\\"actual\\\": \\\"PASS\\\"}, \\\"Test6\\\": {\\\"expected\\\": \\\"PASS TIMEOUT\\\", \\\"actual\\\": \\\"TIMEOUT\\\"}}}, \\\"interrupted\\\": false, \\\"version\\\": 3, \\\"seconds_since_epoch\\\": 14000002, \\\"num_failures_by_type\\\": {\\\"FAIL\\\": 0, \\\"PASS\\\": 2}}\", @@@",
+      "@@@STEP_LOG_LINE@outdir_json@  \"summary.json\": \"{\\\"shards\\\": [{\\\"internal_failure\\\": false, \\\"failure\\\": false, \\\"exit_code\\\": \\\"0\\\"}, {\\\"internal_failure\\\": false, \\\"failure\\\": false, \\\"exit_code\\\": \\\"0\\\"}, {\\\"internal_failure\\\": false, \\\"failure\\\": false, \\\"exit_code\\\": \\\"0\\\"}]}\"@@@",
       "@@@STEP_LOG_LINE@outdir_json@}@@@",
       "@@@STEP_LOG_END@outdir_json@@@",
-      "@@@STEP_LOG_LINE@no_results_exc@error decoding chart JSON results from shard #1@@@",
+      "@@@STEP_LOG_LINE@no_results_exc@Invalid json test results@@@",
       "@@@STEP_LOG_END@no_results_exc@@@",
-      "@@@STEP_LOG_LINE@invalid_results_exc@'failures'@@@",
+      "@@@STEP_LOG_LINE@invalid_results_exc@'valid'@@@",
       "@@@STEP_LOG_END@invalid_results_exc@@@",
       "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
       "@@@STEP_LINK@shard #1@https://chromium-swarm.appspot.com/user/task/10100@@@",
-      "@@@STEP_FAILURE@@@"
+      "@@@STEP_LINK@shard #2@https://chromium-swarm.appspot.com/user/task/10200@@@"
     ]
   },
   {
     "name": "$result",
-    "reason": "1 tests failed: ['telemetry_gpu_unittests']",
     "recipe_result": null,
-    "status_code": 1
+    "status_code": 0

[eyaich1, 2016/10/10 13:51:28]

If this is invalid format should it be returning 0 or 1?

[nednguyen, 2016/10/10 14:42:29]

Oh, this file is newly created, so the diff is for comparing with some existing
expected test file.

Though I think we should never override status_code of the test. If the pipeline
fails & status_code of the test is still zero, the system should report that.

   }
 ]