GTTF: Correctly handle perf tests

scripts/master/factory/chromium_commands.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Set of utilities to add commands to a buildbot factory.
 
 This is based on commands.py and adds chromium-specific commands."""
 
 import logging
 import os
@@ skipping 436 matching lines @@
     # We don't need to run the Reference tests in debug mode.
     if self._target == 'Debug':
       test_list += ':-*.*Ref*'
 
     self.AddAnnotatedPerfStep('startup', test_list, 'graphing',
                               factory_properties=factory_properties)
 
   def AddCCPerfTests(self, factory_properties=None):
     self.AddAnnotatedPerfStep('cc_perftests', None, 'graphing',
                               cmd_name='cc_perftests',
+                              cmd_options=[
+                                  # Retrieving perf test results currently
+                                  # relies on parsing stdout.
+                                  # TODO(phajdan.jr): Do not parse stdout.
+                                  '--test-launcher-print-test-stdio=always'
+                              ],
                               step_name='cc_perftests',
                               factory_properties=factory_properties)
 
   def AddMediaPerfTests(self, factory_properties=None):
     self.AddAnnotatedPerfStep('media_perftests', None, 'graphing',
                               cmd_name='media_perftests',
-                              cmd_options=['--single-process-tests'],

[ghost stip (do not use), 2014/01/09 19:42:00]

don't we still want single-process-tests?

[Paweł Hajdan Jr., 2014/01/10 10:05:47]

No, it's wrong. Generally it shouldn't be used buildbot-side. One notable
exception now are Valgrind tests, I just need to work with Valgrind folks on a
possibly better solution.

--single-process-tests bypasses the new test launcher, and is used internally by
the launcher, and can also be used locally to debug a specific test (as opposed
to the launcher).

I really wonder why this is taking so many round-trips. First, you've asked me
to review the original CL (https://codereview.chromium.org/112633009) and I told
you it's wrong. I added both --single-process-tests and
--test-launcher-print-test-stdio, so presumably I know what's the right way to
use them (and I'm open to feedback, but then please include rationale for using
--single-process-tests stronger than "just in case"). :) Chromium-side change
(https://codereview.chromium.org/128593002/) was unanimously approved within 5
hours.

+                              cmd_options=[
+                                  # Retrieving perf test results currently
+                                  # relies on parsing stdout.
+                                  # TODO(phajdan.jr): Do not parse stdout.
+                                  '--test-launcher-print-test-stdio=always'
+                              ],
                               step_name='media_perftests',
                               factory_properties=factory_properties)
 
   def AddMemoryTests(self, factory_properties=None):
     self.AddAnnotatedPerfStep('memory', 'GeneralMix*MemoryTest.*', 'graphing',
                               factory_properties=factory_properties)
 
   def AddSyncPerfTests(self, factory_properties=None):
     options = ['--ui-test-action-max-timeout=120000']
 
@@ skipping 1110 matching lines @@
   return '%s/%s/%s' % (config.Master.archive_url, archive_type, builder_name)
 
 
 def _GetSnapshotUrl(factory_properties=None, builder_name='%(build_name)s'):
   if not factory_properties or 'gs_bucket' not in factory_properties:
     return (_GetArchiveUrl('snapshots', builder_name), None)
   gs_bucket = factory_properties['gs_bucket']
   gs_bucket = re.sub(r'^gs://', 'http://commondatastorage.googleapis.com/',
                      gs_bucket)
   return ('%s/index.html?path=%s' % (gs_bucket, builder_name), '/')