Retry tests more agressively in nacl_integration.

ppapi/native_client/tools/browser_tester/browser_tester.py

 #!/usr/bin/python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import glob
 import optparse
 import os.path
 import socket
 import sys
@@ skipping 280 matching lines @@
           break
       elif not options.interactive and server.TimedOut(options.timeout):
         js_time = server.TimeSinceJSHeartbeat()
         err = 'Did not hear from the test for %.1f seconds.' % options.timeout
         err += '\nHeard from Javascript %.1f seconds ago.' % js_time
         if js_time > 2.0:
           err += '\nThe renderer probably hung or crashed.'
         else:
           err += '\nThe test probably did not get a callback that it expected.'
         listener.ServerError(err)
+        if not server.received_request:
+          raise RetryTest('Chrome hung before running the test.')
         break
       elif not options.interactive and HardTimeout(options.hard_timeout):
         listener.ServerError('The test took over %.1f seconds.  This is '
                              'probably a runaway test.' % options.hard_timeout)
         break
       else:
         # If Python 2.5 support is dropped, stick server.handle_request() here.
         time.sleep(0.125)
 
     if options.tool:
@@ skipping 38 matching lines @@
 
 # This is an entrypoint for tests that treat the browser tester as a Python
 # library rather than an opaque script.
 # (e.g. run_inbrowser_trusted_crash_in_startup_test)
 def Run(url, options):
   result = 1
   attempt = 1
   while True:
     try:
       result = RunTestsOnce(url, options)
+      if result:
+        # Currently (2013/11/15) nacl_integration is fairly flaky and there is
+        # not enough time to look into it.  Retry if the test fails for any
+        # reason.  Note that in general this test runner tries to only retry
+        # when a known flake is encountered.  (See the other raise
+        # RetryTest(..)s in this file.)  This blanket retry means that those
+        # other cases could be removed without changing the behavior of the test
+        # runner, but it is hoped that this blanket retry will eventually be
+        # unnecessary and subsequently removed.  The more precise retries have
+        # been left in place to preserve the knowledge.
+        raise RetryTest('HACK retrying failed test.')
       break
     except RetryTest:
       # Only retry once.
       if attempt < 2:
         sys.stdout.write('\n@@@STEP_WARNINGS@@@\n')
         sys.stdout.write('WARNING: suspected flake, retrying test!\n\n')
         attempt += 1
         continue
       else:
         sys.stdout.write('\nWARNING: failed too many times, not retrying.\n\n')
@@ skipping 13 matching lines @@
   # Validate the URL
   url = options.url
   if url is None:
     parser.error('Must specify a URL')
 
   return Run(url, options)
 
 
 if __name__ == '__main__':
   sys.exit(RunFromCommandLine())