Adding a startup regression test.

tests/abi_corpus/startup_regression_test.py

Index: tests/abi_corpus/startup_regression_test.py
diff --git a/tests/abi_corpus/startup_regression_test.py b/tests/abi_corpus/startup_regression_test.py
new file mode 100755
index 0000000000000000000000000000000000000000..f9f3c834a1c91b9730098932672939a9b02d7f11
--- /dev/null
+++ b/tests/abi_corpus/startup_regression_test.py
@@ -0,0 +1,191 @@
+#!/usr/bin/python
+# Copyright (c) 2012 The Native Client Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import optparse
+import os
+import re
+import sys
+import tempfile
+
+import corpus_utils
+
+
+BAD_MANIFEST = r'^BAD MANIFEST!$'

[Nick Bray, 2012/04/03 20:34:29]

Gratuitous raw string.

[bradn, 2012/04/12 17:45:54]

Done.

+NACL_MODULE_CRASHED = '^nacl module crashed$'
+NACL_MODULE_DIDNT_START = '^nacl module not started$'
+KNOWN_BAD = {

[Nick Bray, 2012/04/03 20:34:29]

Very optional: you could split this config stuff out as another module.  This
would allow "routine" changes to not touch the main script - the intent of the
change would be immediately obvious.

[bradn, 2012/04/12 17:45:54]

Done.

+  # Bad manifest
+  '0937b653af5553856532454ec340d0e0075bc0b4': BAD_MANIFEST,
+  '09ffe3793113fe564b71800a5844189c00bd8210': BAD_MANIFEST,
+  '14f389a8c406d60e0fc05a1ec0189a652a1f006e': BAD_MANIFEST,
+  '2f97cec9f13b0f774d1f49490f26f32213e4e0a5': BAD_MANIFEST,
+  '3d6832749c8c1346c65b30f4b191930dec5f04a3': BAD_MANIFEST,
+  '612a5aaa821b4b636168025f027e721c0f046e7c': BAD_MANIFEST,
+  '81a4a3de69dd4ad169b1d4a7268b44c78ea5ffa8': BAD_MANIFEST,
+  'a8aa42d699dbef3e1403e4fdc49325e89a91f653': BAD_MANIFEST,
+  'c6d40d4f3c8dccc710d8c09bfd074b2d20a504d2': BAD_MANIFEST,
+  'ced1fea90b71b0a8da08c1a1e6cb35975cc84f52': BAD_MANIFEST,
+  # Bad permissions
+  '8de65668cc7280ffb70ffd2fa5b2a22112156966': (
+      r'^unknown error: :ERROR:extension_error_reporter.cc\(.*\)\] '
+      r'Extension error: Could not load extension from \'.*\'\. '
+      r'Access to permission \'terminalPrivate\' denied\.$'),
+  # No nacl module
+  '1f861c0d8c173b64df3e70cfa1a5cd710ba59430': NACL_MODULE_DIDNT_START,
+  '48ac71edfeac0c79d5e2b651e082abfa76da25f9': NACL_MODULE_DIDNT_START,
+  '4beecff67651f13e013c12a5bf3661041ded323c': NACL_MODULE_DIDNT_START,
+  '57be161e5ff7011d2283e507a70f9005c448002b': NACL_MODULE_DIDNT_START,
+  '5b9f0f7f7401cb666015090908d97346f5b6bccb': NACL_MODULE_DIDNT_START,
+  'cfd62adf6790eed0520da2deb2246fc02e70c57e': NACL_MODULE_DIDNT_START,
+  'd1f33ad38f9f6e78150d30c2103b5c77a9f0adcd': NACL_MODULE_DIDNT_START,
+  'd51802b12a503f7fdfbec60d96e27a5ffa09d002': NACL_MODULE_DIDNT_START,
+  # Nacl module crash
+  'ab0692192976dc6693212582364c6f23b6551d1a': NACL_MODULE_CRASHED,
+  'b458cd57c8b4e6c313b18f370fad59779f573afc': NACL_MODULE_CRASHED,
+  }
+# Precompile the above regexs.
+for key in KNOWN_BAD:
+  KNOWN_BAD[key] = re.compile(KNOWN_BAD[key])
+
+
+def ExpectedResult(path):
+  """Checks what the expected result for this app is.
+
+  Args:
+    path: path to the crx.
+  Returns:
+    Reg-ex that matches the expected status.
+  """
+  return KNOWN_BAD.get(
+      corpus_utils.Sha1FromFilename(path), re.compile('^GOOD$'))
+
+
+def TestAppStartup(options, crx_path, app_path, profile_path):
+  """Run the validator on a nexe, check if the result is expected.
+
+  Args:
+    options: bag of options.
+    crx_path: path to the crx.
+    app_path: path to the extracted crx.
+    profile_path: path to a temporary profile dir.
+  """
+  expected_result = ExpectedResult(crx_path)
+  try:
+    manifest = corpus_utils.LoadManifest(app_path)
+    start_path = manifest.get('app').get('launch').get('local_path')
+  except:
+    if expected_result.match('BAD MANIFEST!'):

[Nick Bray, 2012/04/03 20:34:29]

This is a little sketchy, but I don't have a good suggestion.  Split the config
up, have explicit sets of hashes for each type of failure that doesn't really
require a regex?  Sorta ugly, too.

[bradn, 2012/04/12 17:45:54]

Added a CrxResult class.

+      return True
+    else:
+      print "'%s': 'BAD MANIFEST!'," % corpus_utils.Sha1FromFilename(crx_path)
+      return False
+  start_url = 'chrome-extension://%s/%s' % (
+      corpus_utils.ChromeAppIdFromPath(app_path), start_path)
+  cmd = [options.browser,
+      '--enable-nacl',

[Nick Bray, 2012/04/03 20:34:29]

Check out toosl/browser_tester/browsertester/browserlauncher.py for other
command line arguments you might want.

[bradn, 2012/04/12 17:45:54]

Done.

+      '--load-extension=' + app_path,
+      '--user-data-dir=' + profile_path, start_url]
+  process_stdout, process_stderr, retcode = corpus_utils.RunWithTimeout(
+      cmd, options.duration)
+  # Check for errors we don't like.
+  result = 'GOOD'
+  if 'NaCl process exited with' in process_stderr:
+    result = 'nacl module crashed'
+  errs = re.findall(':ERROR:[^\n]+', process_stderr)
+  if result == 'GOOD':
+    for err in errs:
+      if ('extension_prefs.cc' not in err and
+          'gles2_cmd_decoder.cc' not in err):
+        result = 'unknown error: ' + err
+        break
+  if result == 'GOOD' and 'NaClMakePcrelThunk:' not in process_stderr:
+    result = 'nacl module not started'
+  # Check if result is what we expect.
+  if not expected_result.match(result):
+    print "'%s': '%s'," % (corpus_utils.Sha1FromFilename(crx_path), result)
+    if options.verbose:
+      print '-' * 70
+      print 'Return code: %s' % retcode
+      print '>>> STDOUT'
+      print process_stdout
+      print '>>> STDERR'
+      print process_stderr
+      print '-' * 70
+    return False
+  return True
+
+
+def TestApps(options, work_dir):
+  """Test a browser on a corpus of crxs.
+
+  Args:
+    options: bag of options.
+    work_dir: directory to operate in.
+  """
+  profile_path = os.path.join(work_dir, 'profile_temp')
+  app_path = os.path.join(work_dir, 'app_temp')
+
+  list_filename = os.path.join(work_dir, 'naclapps.all')
+  filenames = corpus_utils.DownloadCorpusTotalList(list_filename)
+  progress = corpus_utils.Progress(len(filenames))
+  for filename in filenames:
+    progress.Tally()
+    corpus_utils.PrimeCache(options, filename)
+    # Stop here if downloading only.
+    if options.download_only:
+      continue
+    # Stop here if isn't a bad app but testing only bad apps.
+    if options.bad_only and ExpectedResult(filename) == 'GOOD':
+      continue
+    # Unzip the app.
+    corpus_utils.ExtractFromCache(options, filename, app_path)
+    try:
+      progress.Result(
+          TestAppStartup(options, filename, app_path, profile_path))
+    finally:
+      corpus_utils.RemoveDir(app_path)
+      corpus_utils.RemoveDir(profile_path)
+  progress.Summary()
+
+
+def Main():
+  if sys.platform in ['cygwin', 'win32']:
+    raise Exception('Platform not yet supported')

[Nick Bray, 2012/04/03 20:34:29]

Comment: why?

[bradn, 2012/04/12 17:45:54]

Done.

+
+  parser = optparse.OptionParser()
+  corpus_utils.SetupOptions(parser)
+  parser.add_option(
+      '--download-only', dest='download_only',
+      default=False, action='store_true',
+      help='download to cache without running the tests')
+  parser.add_option(
+      '--duration', dest='duration', default=30,
+      help='how long to run each app for')
+  parser.add_option(
+      '--browser', dest='browser',
+      help='browser to run')
+  parser.add_option(
+      '-v', '--verbose', dest='verbose', default=False, action='store_true',
+      help='run in verbose mode')
+  parser.add_option(
+      '--bad-only', dest='bad_only', default=False, action='store_true',
+      help='test only known bad apps')
+  options, args = parser.parse_args()
+  if args:
+    parser.error('unused arguments')
+  if not options.download_only:
+    if not options.browser:
+      parser.error('no browser specified')
+
+  work_dir = tempfile.mkdtemp(suffix='startup_crxs', prefix='tmp')
+  work_dir = os.path.realpath(work_dir)
+  try:
+    TestApps(options, work_dir)
+  finally:
+    corpus_utils.RemoveDir(work_dir)
+
+
+if __name__ == '__main__':
+  Main()