Exclude test types that don't upload from the flakiness dashboard UI.

Tools/TestResultServer/handlers/buildershandler.py

 # Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 11 matching lines @@
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import datetime
 import json
 import logging
+import re
 import sys
 import traceback
 import urllib2
 import webapp2
 
 from google.appengine.api import memcache
 
 MASTERS = [
     {'name': 'ChromiumWin', 'url': 'http://build.chromium.org/p/chromium.win', 'groups': ['@ToT Chromium']},
     {'name': 'ChromiumMac', 'url': 'http://build.chromium.org/p/chromium.mac', 'groups': ['@ToT Chromium']},
     {'name': 'ChromiumLinux', 'url': 'http://build.chromium.org/p/chromium.linux', 'groups': ['@ToT Chromium']},
     {'name': 'ChromiumChromiumOS', 'url': 'http://build.chromium.org/p/chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
     {'name': 'ChromiumGPU', 'url': 'http://build.chromium.org/p/chromium.gpu', 'groups': ['@ToT Chromium']},
     {'name': 'ChromiumGPUFYI', 'url': 'http://build.chromium.org/p/chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
     {'name': 'ChromiumWebkit', 'url': 'http://build.chromium.org/p/chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
     {'name': 'ChromiumFYI', 'url': 'http://build.chromium.org/p/chromium.fyi', 'groups': ['@ToT Chromium FYI']},
     {'name': 'V8', 'url': 'http://build.chromium.org/p/client.v8', 'groups': ['@ToT V8']},
 ]
 
+# Buildbot steps that have test in the name, but don't run tests.
+NON_TEST_STEP_NAMES = [
+    'archive',
+    'Run tests',
+    'find isolated tests',
+    'read test spec',
+    'Download latest chromedriver',
+    'compile tests',
+    'create_coverage_',
+    'update test result log',
+    'memory test:',
+    'install_',
+]
+
+# Buildbot steps that run tests but don't upload results to the flakiness dashboard server.
+# FIXME: These should be fixed to upload and then removed from this list.
+TEST_STEPS_THAT_DO_NOT_UPLOAD_YET = [
+    'java_tests(chrome',
+    'python_tests(chrome',
+    'run_all_tests.py',
+    'test_report',
+    'test CronetSample',
+    'test_mini_installer',
+    'telemetry_unittests',
+    'webkit_python_tests',
+    'webkit_unit_tests',
+]
 
 class FetchBuildersException(Exception): pass
 
 
 def master_json_url(master_url):
     return master_url + '/json/builders'
 
 
 def builder_json_url(master_url, builder):
     return master_json_url(master_url) + '/' + urllib2.quote(builder)
@@ skipping 74 matching lines @@
             latest_build = get_latest_build(build_data)
             if not latest_build:
                 logging.info('Skipping builder %s because it lacked cached or current builds.', builder)
                 continue
 
             build = fetch_json(cached_build_json_url(master_url, builder, latest_build))
             if not build:
                 logging.info('Skipping build %s on builder %s due to empty data', latest_build, builder)
             for step in build['steps']:
                 step_name = step['name']
-                is_test_step = 'test' in step_name and 'archive' not in step_name and 'Run tests' not in step_name
+
+                is_test_step = 'test' in step_name
+                for name in NON_TEST_STEP_NAMES:

[Dirk Pranke, 2014/06/09 19:06:12]

to replace lines 174-183:

if not 'test' in step_name:
  continue

if any(n in step_name for n in NON_TEST_STEP_NAMES):
  continue

if re.search(...):
  continue

? 
                                                                         

[ojan, 2014/06/09 21:39:16]

Indeed! Also, I realized I forgot to run the unittests. Updated the unittest to
make sure to hit each continue in the new code.

+                    if name in step_name:
+                        is_test_step = False
+                        break
+
+                # Hackity hack. But there's not a better way to exclude certain test steps without
+                # explicitly listing them all. Maybe that would be better?
+                if is_test_step and re.search('/_only|_ignore|_perf$/', step_name):
+                    is_test_step = False
+
                 if not is_test_step:
                     continue
 
                 if step_name == 'webkit_tests':
                     step_name = 'layout-tests'
 
                 tests_object.setdefault(step_name, {'builders': []})
                 tests_object[step_name]['builders'].append(builder)
 
         for builders in tests_object.values():
             builders['builders'].sort()
 
-    output_data = {'masters': master_data}
+    output_data = {'masters': master_data, 'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET}
 
     delta = datetime.datetime.now() - start_time
 
     logging.info('Fetched buildbot data in %s seconds.', delta.seconds)
 
     return dump_json(output_data)
 
 
 class UpdateBuilders(webapp2.RequestHandler):
     """Fetch and update the cached buildbot data."""
@@ skipping 25 matching lines @@
             buildbot_data = fetch_buildbot_data(MASTERS, True)
             try:
                 memcache.set('buildbot_data', buildbot_data)
             except ValueError, err:
                 logging.error(str(err))
 
         if callback:
             buildbot_data = callback + '(' + buildbot_data + ');'
 
         self.response.out.write(buildbot_data)