Retry tests more agressively in nacl_integration.

ppapi/native_client/tools/browser_tester/browser_tester.py

 #!/usr/bin/python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import glob
 import optparse
 import os.path
 import socket
 import sys
@@ skipping 280 matching lines @@
           break
       elif not options.interactive and server.TimedOut(options.timeout):
         js_time = server.TimeSinceJSHeartbeat()
         err = 'Did not hear from the test for %.1f seconds.' % options.timeout
         err += '\nHeard from Javascript %.1f seconds ago.' % js_time
         if js_time > 2.0:
           err += '\nThe renderer probably hung or crashed.'
         else:
           err += '\nThe test probably did not get a callback that it expected.'
         listener.ServerError(err)
+        if not server.received_request:
+          raise RetryTest('Chrome hung before running the test.')
         break
       elif not options.interactive and HardTimeout(options.hard_timeout):
         listener.ServerError('The test took over %.1f seconds.  This is '
                              'probably a runaway test.' % options.hard_timeout)
         break
       else:
         # If Python 2.5 support is dropped, stick server.handle_request() here.
         time.sleep(0.125)
 
     if options.tool:
@@ skipping 38 matching lines @@
 
 # This is an entrypoint for tests that treat the browser tester as a Python
 # library rather than an opaque script.
 # (e.g. run_inbrowser_trusted_crash_in_startup_test)
 def Run(url, options):
   result = 1
   attempt = 1
   while True:
     try:
       result = RunTestsOnce(url, options)
+      if result and options.tool is None:

[Mark Seaborn, 2013/11/14 01:27:58]

To be honest, I don't understand why you're adding two new 'raise RetryTest'
cases.  Doesn't this one retry on all failures, so that the first wouldn't be
necessary?

Also, I don't understand why you check options.tool here but not above.

+        raise RetryTest('HACK retrying failed test.')
       break
     except RetryTest:
       # Only retry once.
       if attempt < 2:
         sys.stdout.write('\n@@@STEP_WARNINGS@@@\n')
         sys.stdout.write('WARNING: suspected flake, retrying test!\n\n')
         attempt += 1
         continue
       else:
         sys.stdout.write('\nWARNING: failed too many times, not retrying.\n\n')
@@ skipping 13 matching lines @@
   # Validate the URL
   url = options.url
   if url is None:
     parser.error('Must specify a URL')
 
   return Run(url, options)
 
 
 if __name__ == '__main__':
   sys.exit(RunFromCommandLine())