Add option to command tester for running a test multiple times. Use that

tests/browser_startup_time/nacl.scons

 # -*- python -*-
 # Copyright (c) 2011 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Test startup time for loading a basic PPAPI nexe in the browser.
 
 Import('env')
 
-# Pull in the ppapi_test_example as a "barebones" PPAPI startup example.
-# Hope it does not change to do more than just the barebones.
-# We must make a separate build though, to get a different nexe "key"
-# for the env.Publish() / env.ExtractPublishedFiles() magic.
 nexe_name = 'browser_startup_time_${TARGET_FULLARCH}'
 stripped_nexe = ('browser_startup_time_stripped_%s.nexe' %
                  env.get('TARGET_FULLARCH'))
 
 browser_startup_nexe = env.ComponentProgram(
     nexe_name,
     ['hello_world.cc'],
     EXTRA_LIBS=['${PPAPI_LIBS}', 'ppapi_cpp'])
 
 env.Publish(nexe_name, 'run',
             ['browser_startup_time.html',
              'browser_startup_time.js'])
 
 test_name = 'browser_startup_time.out'
-output_processor = ['${PYTHON}',
-                    str(env.File(
-                        '${SCONSTRUCT_DIR}/tools/process_perf_output.py')),
-                    test_name,
-                    env.GetPerfEnvDescription()]
+output_processor_1 = ['${PYTHON}',
+                      str(env.File(
+                          '${SCONSTRUCT_DIR}/tools/process_perf_output.py')),
+                      test_name,
+                      env.GetPerfEnvDescription()]
+output_processor_c = ['${PYTHON}',
+                      str(env.File(
+                          '${SCONSTRUCT_DIR}/tools/process_perf_combined.py'))]
+
 
 node = env.PPAPIBrowserTester(
     test_name,
     url='browser_startup_time.html',
     nmfs=['browser_startup_time.nmf'],
     files=env.ExtractPublishedFiles(nexe_name),
-    args=['--enable_experimental_js'],
-    process_output=output_processor,
+    # Smooth out the timing data.
+    num_runs='5',

[Nick Bray, 2011/11/07 23:39:09]

This parameter should be an integer, at least until it is passed on the command
line.

[jvoung - send to chromium..., 2011/11/09 00:50:54]

I'm seeing some weird stuff:


run_browser_startup_time_test
(scons-out/nacl-x86-32/test_results/browser_startup_time.out) failed:
AttributeError `'int' object has no attribute 'abspath'' trying to evaluate
`${SOURCES[1].abspath}'

when I do that...

[jvoung - send to chromium..., 2011/11/09 01:00:38]

Same thing happens if I go into SConstruct and change this code:

  # Other extra flags
  if not capture_output:
    script_flags.extend(['--capture_output', '0'])

from '0' to 0.

Maybe the command arguments are being analyzed as "sources".

So... can I leave this one as a string for now?

[jvoung - send to chromium..., 2011/11/09 17:42:53]

Okay, fixed. Is there a better way to detect non-strings and non-Scons objects
in python? Currently only converting ints to strings.

+    process_output_single=output_processor_1,
+    process_output_combined=output_processor_c,
     # Assume verbosity level of 1 will give us NaClLogs for sel_ldr times.
     log_verbosity=1)
 
 # TODO(jvoung): use a single test suite when either
 # (a) performance_tests are always run, or
 # (b) we have decided to just fold this test into chrome_browser_tests.
 test_suites = ['chrome_browser_tests', 'performance_tests']
 env.AddNodeToTestSuite(node,
                        test_suites,
                        'run_browser_startup_time_test',
@@ skipping 12 matching lines @@
 
 # Check size of stripped nexe.
 browser_startup_stripped_nexe = env.StripExecutable(
     stripped_nexe,
     browser_startup_nexe)
 node = env.FileSizeTest('browser_startup_stripped_size.out',
                         browser_startup_stripped_nexe)
 env.AddNodeToTestSuite(node,
                        test_suites,
                        'run_browser_startup_stripped_size_test')