Cap the maximum number of flaky tests that we record from a step

appengine/chromium_try_flakes/handlers/flake_issues.py

Index: appengine/chromium_try_flakes/handlers/flake_issues.py
diff --git a/appengine/chromium_try_flakes/handlers/flake_issues.py b/appengine/chromium_try_flakes/handlers/flake_issues.py
index dd158c2f3b83b6cf8c2f5e2115ffc7e231cf924e..371cdd2c6b8f30bcef36dfadeae1cc8ad09a68ca 100644
--- a/appengine/chromium_try_flakes/handlers/flake_issues.py
+++ b/appengine/chromium_try_flakes/handlers/flake_issues.py
@@ -30,6 +30,7 @@ MIN_REQUIRED_FLAKY_RUNS = 3
 DAYS_TILL_STALE = 30
 USE_MONORAIL = True
 DAYS_TO_REOPEN_ISSUE = 3
+MAX_INDIVIDUAL_FLAKES_PER_STEP = 50
 FLAKY_RUNS_TEMPLATE = (
     'Detected %(new_flakes_count)d new flakes for test/step "%(name)s". To see '
     'the actual flakes, please visit %(flakes_url)s. This message was posted '
@@ -508,6 +509,8 @@ class CreateFlakyRun(webapp2.RequestHandler):
         _, failed, _ = cls._flatten_tests(
             json_result.get('tests', {}),
             json_result.get('path_delimiter', '/'))
+        if len(failed) > MAX_INDIVIDUAL_FLAKES_PER_STEP:
+          return [stepname], True
         return failed, False
 
       if result.status_code == 404: