Refactor stages 1, 2 and test_api overhaul.

scripts/slave/recipes/bisection/desktop_bisect.expected/basic_perf_tryjob_with_revisions.json

 [
   {
     "cmd": [
       "python",
       "-u",
       "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
       "[BUILDER_CACHE]/linux_perf_bisect",
       "511"
     ],
     "name": "makedirs checkout path",
@@ skipping 276 matching lines @@
       "--source",
       "[BUILDER_CACHE]/linux_perf_bisect/src/tools/run-perf-test.cfg",
       "--output_json",
       "/path/to/tmp/json"
     ],
     "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
     "name": "load config",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@json.output@{@@@",
       "@@@STEP_LOG_LINE@json.output@  \"bad_revision\": \"306476\", @@@",
-      "@@@STEP_LOG_LINE@json.output@  \"command\": \"./tools/perf/run_benchmark -v --browser=release sunspider\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"command\": \"./tools/perf/run_benchmark -v --browser=release sunspider --output_format=valueset\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"good_revision\": \"306475\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"max_time_minutes\": \"25\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"metric\": \"dummy/dummy\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"repeat_count\": \"1\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"target_arch\": \"ia32\", @@@",
       "@@@STEP_LOG_LINE@json.output@  \"truncate_percent\": \"25\"@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
@@ skipping 301 matching lines @@
       "{\"blamelist\": [\"cool_dev1337@chromium.org\", \"hax@chromium.org\"], \"branch\": \"\", \"buildername\": \"linux_perf_bisect\", \"buildnumber\": 571, \"is_test\": true, \"issue\": \"12345\", \"mastername\": \"tryserver.chromium.perf\", \"patch_project\": \"chromium\", \"patch_storage\": \"rietveld\", \"patchset\": \"20001\", \"path_config\": \"kitchen\", \"project\": \"chrome\", \"recipe\": \"bisection/desktop_bisect\", \"repository\": \"\", \"requester\": \"commit-bot@chromium.org\", \"revision\": \"HEAD\", \"rietveld\": \"https://codereview.chromium.org\", \"slavename\": \"TestSlavename\", \"workdir\": \"/path/to/workdir/TestSlavename\"}",
       "--step-name=Performance Test (bad49c331def2a3bbf3ddd0096eb51551155) 1 of 1",
       "--builder-name=linux_perf_bisect",
       "--slave-name=TestSlavename",
       "--build-number=571",
       "--run-python-script",
       "[BUILDER_CACHE]/linux_perf_bisect/src/tools/perf/run_benchmark",
       "-v",
       "--browser=release",
       "sunspider",
+      "--output_format=valueset",
       "--reset-results",
       "--upload-results",
-      "--results-label=bad49c331def2a3bbf3ddd0096eb51551155"
+      "--results-label=bad49c331def2a3bbf3ddd0096eb51551155",
+      "--output-dir=[TMP_BASE]/perf-test-output_tmp_1"
     ],
     "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
     "name": "Performance Test (bad49c331def2a3bbf3ddd0096eb51551155) 1 of 1",
     "stderr": "/path/to/tmp/",
     "stdout": "/path/to/tmp/",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@Captured Output@*RESULT dummy: dummy= [5.83,6.013,5.573]ms@@@",
       "@@@STEP_LOG_LINE@Captured Output@Avg dummy: 5.907711ms@@@",
       "@@@STEP_LOG_LINE@Captured Output@Sd  dummy: 0.255921ms@@@",
       "@@@STEP_LOG_LINE@Captured Output@RESULT telemetry_page_measurement_results: num_failed= 0 count@@@",
       "@@@STEP_LOG_LINE@Captured Output@RESULT telemetry_page_measurement_results: num_errored= 0 count@@@",
       "@@@STEP_LOG_LINE@Captured Output@@@@",
       "@@@STEP_LOG_LINE@Captured Output@View online at http://storage.googleapis.com/chromium-telemetry/html-results/results-with_patch@@@",
       "@@@STEP_LOG_END@Captured Output@@@"
     ]
   },
   {
     "cmd": [
       "python",
       "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "*RESULT dummy: dummy= [5.83,6.013,5.573]ms\nAvg dummy: 5.907711ms\nSd  dummy: 0.255921ms\nRESULT telemetry_page_measurement_results: num_failed= 0 count\nRESULT telemetry_page_measurement_results: num_errored= 0 count\n\nView online at http://storage.googleapis.com/chromium-telemetry/html-results/results-with_patch\n",
+      "[TMP_BASE]/perf-test-output_tmp_1/results.txt"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "write buildbot output to disk"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[TMP_BASE]/perf-test-output_tmp_1/results-valueset.json",
+      "/path/to/tmp/json"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "Reading valueset results",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"dummy\": \"dict\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
       "--spec",
       "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}, {'custom_deps': {'src/chrome/test/data/firefox2_profile/searchplugins': None, 'src/chrome/test/data/firefox2_searchplugins': None, 'src/chrome/test/data/firefox3_profile/searchplugins': None, 'src/chrome/test/data/firefox3_searchplugins': None, 'src/chrome/test/data/osdd': None, 'src/chrome/test/data/perf/frame_rate/private': None, 'src/chrome/test/data/plugin': None, 'src/chrome/test/data/ssl/certs': None, 'src/data/autodiscovery': None, 'src/data/esctf': None, 'src/data/mach_ports': None, 'src/data/memory_test': None, 'src/data/mozilla_js_tests': None, 'src/data/selenium_core': None, 'src/data/tab_switching': None, 'src/tools/grit/grit/test/data': None, 'src/webkit/data/bmp_decoder': None, 'src/webkit/data/ico_decoder': None, 'src/webkit/data/test_shell/plugins': None, 'src/webkit/data/xbm_decoder': None}, 'deps_file': '.DEPS.git', 'managed': False, 'name': 'src-internal', 'url': 'https://chrome-internal.googlesource.com/chrome/src-internal.git'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'catapult', 'url': 'https://chromium.googlesource.com/external/github.com/catapult-project/catapult.git'}]",
       "--root",
       "src",
       "--revision_mapping_file",
       "{\"src\": \"got_revision\", \"src/buildtools\": \"got_buildtools_revision\", \"src/native_client\": \"got_nacl_revision\", \"src/third_party/angle\": \"got_angle_revision\", \"src/third_party/webrtc\": \"got_webrtc_revision\", \"src/tools/swarming_client\": \"got_swarming_client_revision\", \"src/v8\": \"got_v8_revision\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
       "--rietveld_server",
@@ skipping 158 matching lines @@
       "{\"blamelist\": [\"cool_dev1337@chromium.org\", \"hax@chromium.org\"], \"branch\": \"\", \"buildername\": \"linux_perf_bisect\", \"buildnumber\": 571, \"is_test\": true, \"issue\": \"12345\", \"mastername\": \"tryserver.chromium.perf\", \"patch_project\": \"chromium\", \"patch_storage\": \"rietveld\", \"patchset\": \"20001\", \"path_config\": \"kitchen\", \"project\": \"chrome\", \"recipe\": \"bisection/desktop_bisect\", \"repository\": \"\", \"requester\": \"commit-bot@chromium.org\", \"revision\": \"HEAD\", \"rietveld\": \"https://codereview.chromium.org\", \"slavename\": \"TestSlavename\", \"workdir\": \"/path/to/workdir/TestSlavename\"}",
       "--step-name=Performance Test (d49c331def2a3bbf3ddd0096eb51551155) 1 of 1",
       "--builder-name=linux_perf_bisect",
       "--slave-name=TestSlavename",
       "--build-number=571",
       "--run-python-script",
       "[BUILDER_CACHE]/linux_perf_bisect/src/tools/perf/run_benchmark",
       "-v",
       "--browser=release",
       "sunspider",
+      "--output_format=valueset",
       "--upload-results",
-      "--results-label=d49c331def2a3bbf3ddd0096eb51551155"
+      "--results-label=d49c331def2a3bbf3ddd0096eb51551155",
+      "--output-dir=[TMP_BASE]/perf-test-output_tmp_2"
     ],
     "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
     "name": "Performance Test (d49c331def2a3bbf3ddd0096eb51551155) 1 of 1",
     "stderr": "/path/to/tmp/",
     "stdout": "/path/to/tmp/",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@Captured Output@*RESULT dummy: dummy= [5.83,6.013,5.573]ms@@@",
       "@@@STEP_LOG_LINE@Captured Output@Avg dummy: 5.907711ms@@@",
       "@@@STEP_LOG_LINE@Captured Output@Sd  dummy: 0.255921ms@@@",
       "@@@STEP_LOG_LINE@Captured Output@RESULT telemetry_page_measurement_results: num_failed= 0 count@@@",
       "@@@STEP_LOG_LINE@Captured Output@RESULT telemetry_page_measurement_results: num_errored= 0 count@@@",
       "@@@STEP_LOG_LINE@Captured Output@@@@",
       "@@@STEP_LOG_LINE@Captured Output@View online at http://storage.googleapis.com/chromium-telemetry/html-results/results-without_patch@@@",
       "@@@STEP_LOG_END@Captured Output@@@"
     ]
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "*RESULT dummy: dummy= [5.83,6.013,5.573]ms\nAvg dummy: 5.907711ms\nSd  dummy: 0.255921ms\nRESULT telemetry_page_measurement_results: num_failed= 0 count\nRESULT telemetry_page_measurement_results: num_errored= 0 count\n\nView online at http://storage.googleapis.com/chromium-telemetry/html-results/results-without_patch\n",
+      "[TMP_BASE]/perf-test-output_tmp_2/results.txt"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "write buildbot output to disk (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[TMP_BASE]/perf-test-output_tmp_2/results-valueset.json",
+      "/path/to/tmp/json"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "Reading valueset results (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"dummy\": \"dict\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/catapult/tracing/bin/parse_metric_cmdline",
+      "[TMP_BASE]/perf-test-output_tmp_1/results-valueset.json",
+      "dummy/dummy",
+      "--valueset"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "Parse metric values",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@[@@@",
+      "@@@STEP_LOG_LINE@json.output@  1, @@@",
+      "@@@STEP_LOG_LINE@json.output@  1, @@@",
+      "@@@STEP_LOG_LINE@json.output@  1@@@",
+      "@@@STEP_LOG_LINE@json.output@]@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/catapult/tracing/bin/parse_metric_cmdline",
+      "[TMP_BASE]/perf-test-output_tmp_2/results-valueset.json",
+      "dummy/dummy",
+      "--valueset"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "Parse metric values (2)",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@[@@@",
+      "@@@STEP_LOG_LINE@json.output@  9, @@@",
+      "@@@STEP_LOG_LINE@json.output@  9, @@@",
+      "@@@STEP_LOG_LINE@json.output@  9@@@",
+      "@@@STEP_LOG_LINE@json.output@]@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
     "cmd": [],
     "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
     "name": "Results",
     "~followup_annotations": [
       "@@@STEP_TEXT@",
       "===== PERF TRY JOB RESULTS =====",
       "",
-      "Test Command: src/tools/perf/run_benchmark -v --browser=release sunspider",
+      "Test Command: src/tools/perf/run_benchmark -v --browser=release sunspider --output_format=valueset",
       "Test Metric: dummy/dummy",
-      "Relative Change: 0.00000%",
-      "Standard Error: +- 0.18047 delta",
+      "Relative Change: 88.88889%",
+      "Standard Error: +- 0.00000 delta",
       "",
       "Revision    Mean        Std.Error",
-      "Patch       5.805333333330.127614436662",
-      "No Patch    5.805333333330.127614436662",
+      "Patch       1.0         0.0",
+      "No Patch    9.0         0.0",
       "@@@",
       "@@@STEP_LINK@HTML Results@http://storage.googleapis.com/chromium-telemetry/html-results/results-without_patch@@@"
     ]
   },
   {
     "cmd": [
       "python",
       "-u",
+      "[SLAVE_BUILD]/catapult/tracing/bin/parse_metric_cmdline",
+      "[TMP_BASE]/perf-test-output_tmp_1/results-valueset.json",
+      "dummy/dummy",
+      "--valueset"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "Parse metric values (3)",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@[@@@",
+      "@@@STEP_LOG_LINE@json.output@  1, @@@",
+      "@@@STEP_LOG_LINE@json.output@  1, @@@",
+      "@@@STEP_LOG_LINE@json.output@  1@@@",
+      "@@@STEP_LOG_LINE@json.output@]@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/catapult/tracing/bin/parse_metric_cmdline",
+      "[TMP_BASE]/perf-test-output_tmp_2/results-valueset.json",
+      "dummy/dummy",
+      "--valueset"
+    ],
+    "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
+    "name": "Parse metric values (4)",
+    "stdout": "/path/to/tmp/json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@[@@@",
+      "@@@STEP_LOG_LINE@json.output@  9, @@@",
+      "@@@STEP_LOG_LINE@json.output@  9, @@@",
+      "@@@STEP_LOG_LINE@json.output@  9@@@",
+      "@@@STEP_LOG_LINE@json.output@]@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "RECIPE_MODULE[build::perf_dashboard]/resources/post_json.py"
     ],
     "cwd": "[BUILDER_CACHE]/linux_perf_bisect",
     "name": "Post bisect results",
-    "stdin": "{\"data\": {\"bisect_bot\": \"linux_perf_bisect\", \"buildbot_log_url\": \"http://c.org/p/cr/builders/linux_perf_bisect/builds/571\", \"change\": 0.0, \"cloud_link\": \"http://storage.googleapis.com/chromium-telemetry/html-results/results-without_patch\", \"command\": \"src/tools/perf/run_benchmark -v --browser=release sunspider\", \"metric\": \"dummy/dummy\", \"profiler_links\": [], \"result\": \"Revision    Mean        Std.Error\\nPatch       5.805333333330.127614436662\\nNo Patch    5.805333333330.127614436662\", \"status\": \"completed\", \"std_err\": 0.1804740670813644, \"try_job_id\": null}, \"url\": \"https://chromeperf.appspot.com/post_bisect_results\"}",
+    "stdin": "{\"data\": {\"bisect_bot\": \"linux_perf_bisect\", \"buildbot_log_url\": \"http://c.org/p/cr/builders/linux_perf_bisect/builds/571\", \"change\": 88.88888888888889, \"cloud_link\": \"http://storage.googleapis.com/chromium-telemetry/html-results/results-without_patch\", \"command\": \"src/tools/perf/run_benchmark -v --browser=release sunspider --output_format=valueset\", \"metric\": \"dummy/dummy\", \"profiler_links\": [], \"result\": \"Revision    Mean        Std.Error\\nPatch       1.0         0.0\\nNo Patch    9.0         0.0\", \"status\": \"completed\", \"std_err\": 0.0, \"try_job_id\": null}, \"url\": \"https://chromeperf.appspot.com/post_bisect_results\"}",
     "stdout": "/path/to/tmp/json",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@json.output@{@@@",
       "@@@STEP_LOG_LINE@json.output@  \"status_code\": 200@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
   }
 ]