Disable flaky BenchmarkSmokeTest.thread_times.simple_mobile_sites

tools/perf/benchmarks/benchmark_unittest.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Run the first page of every benchmark that has a composable measurement.
 
 Ideally this test would be comprehensive, but the above serves as a
 kind of smoke test.
 """
 
 import os
 import unittest
 
 from telemetry import benchmark as benchmark_module
 from telemetry.core import discover
 from telemetry.page import page_measurement
 from telemetry.unittest import options_for_unittests
 from telemetry.unittest import output_formatter
 
 
 def SmokeTestGenerator(benchmark):
   # In general you should @benchmark_module.Disabled individual benchmarks that
   # fail, instead of this entire smoke test suite.
   # TODO(achuith): Multiple tests failing on CrOS. crbug.com/351114
   @benchmark_module.Disabled('chromeos')
+  # Flaky, http://crbug.com/400747 .
+  @benchmark_module.Disabled('mac')

[tonyg, 2014/08/05 15:19:46]

This isn't the right fix for this. Please see comment above.

Instead we should disable the thread_times.simple_mobile_sites benchmark.

   def BenchmarkSmokeTest(self):
     # Only measure a single page so that this test cycles reasonably quickly.
     benchmark.options['pageset_repeat'] = 1
     benchmark.options['page_repeat'] = 1
 
     class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
       def CreatePageSet(self, options):
         # pylint: disable=E1002
         ps = super(SinglePageBenchmark, self).CreatePageSet(options)
         ps.pages = ps.pages[:1]
@@ skipping 45 matching lines @@
     if hasattr(benchmark, 'generated_profile_archive'):
       # We'd like to test these, but don't know how yet.
       continue
 
     class BenchmarkSmokeTest(unittest.TestCase):
       pass
     setattr(BenchmarkSmokeTest, benchmark.Name(), SmokeTestGenerator(benchmark))
     suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
 
   return suite