Add option to command tester for running a test multiple times. Use that

SConstruct

 #! -*- python -*-
 # Copyright (c) 2011 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import atexit
 import glob
 import os
 import platform
 import shutil
@@ skipping 1522 matching lines @@
     filter_group_only = params.get('filter_group_only', False)
     post_actions.append(
         GoldenFileCheckAction(
             env, stream_file, golden_file,
             filter_regex, filter_inverse, filter_group_only))
 
   if ShouldUseVerboseOptions(extra):
     env.MakeVerboseExtraOptions(target, log_verbosity, extra)
   # Heuristic for when to capture output...
   capture_output = (extra.pop('capture_output', False)
-                    or 'process_output' in extra)
+                    or 'process_output_single' in extra)
   node = env.CommandTest(target,
                          command,
                          # Set to 'huge' so that the browser tester's timeout
                          # takes precedence over the default of the test_suite.
                          size='huge',
                          capture_output=capture_output,
                          **extra)
   # We can't check output if the test is not run.
   if not env.Bit('do_not_run_tests'):
     for action in post_actions:
@@ skipping 251 matching lines @@
   log_file = env.MakeNaClLogOption(target)
   extra['log_file'] = log_file
   extra_env = ['NACLLOG=%s' % log_file,
                'NACLVERBOSITY=%d' % log_verbosity]
   extra['osenv'] = extra.get('osenv', []) + extra_env
 
 pre_base_env.AddMethod(MakeVerboseExtraOptions)
 
 def ShouldUseVerboseOptions(extra):
   """ Heuristic for setting up Verbose NACLLOG options. """
-  return ('process_output' in extra or
+  return ('process_output_single' in extra or
           'log_golden' in extra)
 
 # ----------------------------------------------------------
 DeclareBit('tests_use_irt', 'Non-browser tests also load the IRT image', False)
 
 def CommandSelLdrTestNacl(env, name, nexe,
                           args = None,
                           log_verbosity=2,
                           sel_ldr_flags=None,
                           loader=None,
@@ skipping 59 matching lines @@
     env.Alias('irt_tests', node)
   return node
 
 pre_base_env.AddMethod(CommandSelLdrTestNacl)
 
 # ----------------------------------------------------------
 TEST_EXTRA_ARGS = ['stdin', 'log_file',
                    'stdout_golden', 'stderr_golden', 'log_golden',
                    'filter_regex', 'filter_inverse', 'filter_group_only',
                    'osenv', 'arch', 'subarch', 'exit_status', 'track_cmdtime',
-                   'process_output', 'using_nacl_signal_handler']
+                   'num_runs', 'process_output_single',
+                   'process_output_combined', 'using_nacl_signal_handler']
 
 TEST_TIME_THRESHOLD = {
     'small':   2,
     'medium': 10,
     'large':  60,
     'huge': 1800,
     }
 
 # Valgrind handles SIGSEGV in a way our testing tools do not expect.
 UNSUPPORTED_VALGRIND_EXIT_STATUS = ['sigabrt',
@@ skipping 91 matching lines @@
     assert flag_name in TEST_EXTRA_ARGS, repr(flag_name)
     if isinstance(flag_value, list):
       # Options to command_tester.py which are actually lists must not be
       # separated by whitespace. This stringifies the lists with a separator
       # char to satisfy command_tester.
       flag_value =  command_tester.StringifyList(flag_value)
     # do not add --flag + |flag_name| |flag_value| if
     # |flag_value| is false (empty).
     if flag_value:
       script_flags.append('--' + flag_name)
+      # Make sure flag values are strings (or SCons objects) when building
+      # up the command. Right now, this only means convert ints to strings.
+      if type(flag_value) == int:

[Nick Bray, 2011/11/11 22:42:39]

isinstance(flag_value, int)

[jvoung - send to chromium..., 2011/11/11 23:28:27]

Done.

+        flag_value = str(flag_value)
       script_flags.append(flag_value)
 
   # Other extra flags
   if not capture_output:
     script_flags.extend(['--capture_output', '0'])
 
   test_script = env.File('${SCONSTRUCT_DIR}/tools/command_tester.py')
   command = ['${PYTHON}', test_script] + script_flags + command
   return AutoDepsCommand(env, name, command,
                          extra_deps=extra_deps, posix_path=posix_path,
@@ skipping 1626 matching lines @@
   nacl_env.ValidateSdk()
 
 if BROKEN_TEST_COUNT > 0:
   msg = "There are %d broken tests." % BROKEN_TEST_COUNT
   if GetOption('brief_comstr'):
     msg += " Add --verbose to the command line for more information."
   print msg
 
 # separate warnings from actual build output
 Banner('B U I L D - O U T P U T:')