Handle and surface application test failures.

mojo/tools/apptest_runner.py

Index: mojo/tools/apptest_runner.py
diff --git a/mojo/tools/apptest_runner.py b/mojo/tools/apptest_runner.py
index d84dd234af44f0eaf755d47211a7f703f39ad357..f8d7df30465fc0492bdf939e03edb44b6d32f1dd 100755
--- a/mojo/tools/apptest_runner.py
+++ b/mojo/tools/apptest_runner.py
@@ -15,6 +15,56 @@ _logging = logging.getLogger()
 
 from mopy.gtest_list_tests import gtest_list_tests
 
+
+def print_output(command_line, output):
+  print command_line
+  print 72 * '-'
+  print output
+  print 72 * '-'
+
+
+def try_command_line(command_line):
+  """Returns the output of a command line or an empty string on error."""
+  _logging.debug("Running command line: %s" % command_line)
+  try:
+    output = subprocess.check_output(command_line, stderr=subprocess.STDOUT)
+    return output
+  except subprocess.CalledProcessError as e:
+    print "Failed with exit code %d, command line, and output:" % e.returncode
+    print_output(command_line, e.output)
+  except Exception as e:
+    print "Failed with command line and exception:"
+    print_output(command_line, e)
+  return None
+
+
+def get_fixtures(command_line):
+  """Returns the "Test.Fixture" list from a --gtest_list_tests commandline."""
+  list_output = try_command_line(command_line)
+  _logging.debug("Tests listed:\n%s" % list_output)
+  try:
+    return gtest_list_tests(list_output)
+  except Exception as e:
+    print "Failed to get test fixtures with command line and exception:"
+    print_output(command_line, e)
+  return []

[viettrungluu, 2014/11/24 20:22:58]

I'd indent this one more level, since it's only used in the except ...: case.

[msw, 2014/11/24 20:46:54]

Done.

+
+
+def run_test(command_line):
+  """Runs a command line and checks the output for signs of gtest failure."""
+  output = try_command_line(command_line)
+  # Fail on output with gtest's "[  FAILED  ]" or a lack of "[  PASSED  ]".
+  # The latter condition ensures failure on broken command lines or output.
+  # Check output instead of exit codes because mojo_shell always exits with 0.
+  if output.find("[  FAILED  ]") != -1 or output.find("[  PASSED  ]") == -1:
+    print "Failed test with command line and output:"
+    print_output(command_line, output)
+    return False
+  else:

[viettrungluu, 2014/11/24 20:22:57]

nit: No else needed.

[msw, 2014/11/24 20:46:54]

Done.

+    _logging.debug("Succeeded with output:\n%s" % output)
+  return True
+
+
 def main(argv):
   logging.basicConfig()
   # Uncomment to debug:
@@ -40,49 +90,28 @@ def main(argv):
   if platform.system() == 'Windows':
     mojo_shell += ".exe"
 
+  exit_code = 0
   for apptest in apptest_list:
     print "Running " + apptest + "...",
     sys.stdout.flush()
 
     # List the apptest fixtures so they can be run independently for isolation.
     # TODO(msw): Run some apptests without fixture isolation?
-    list_command_line = [mojo_shell, apptest + " --gtest_list_tests"]
-    _logging.debug("Listing tests: %s" % list_command_line)
-    try:
-      # TODO(msw): Need to fail on errors! mojo_shell always exits with 0!
-      list_output = subprocess.check_output(list_command_line)
-      _logging.debug("Tests listed:\n%s" % list_output)
-    except subprocess.CalledProcessError as e:
-      print "Failed with exit code %d and output:" % e.returncode
-      print 72 * '-'
-      print e.output
-      print 72 * '-'
-      return 1
-    except OSError as e:
-      print "  Failed to start test"
-      return 1
-
-    test_list = gtest_list_tests(list_output)
-    for test in test_list:
-      command_line = [mojo_shell, apptest + " --gtest_filter=" + test]
-
-      _logging.debug("Running %s..." % command_line)
-      try:
-        # TODO(msw): Need to fail on errors! mojo_shell always exits with 0!
-        subprocess.check_output(command_line, stderr=subprocess.STDOUT)
-        _logging.debug("Succeeded")
-      except subprocess.CalledProcessError as e:
-        print "Failed with exit code %d and output:" % e.returncode
-        print 72 * '-'
-        print e.output
-        print 72 * '-'
-        return 1
-      except OSError as e:
-        print "  Failed to start test"
-        return 1
-    print "Succeeded"
-
-  return 0
+    fixtures = get_fixtures([mojo_shell, apptest + " --gtest_list_tests"])
+
+    if not fixtures:
+      print "Failed with no tests found."
+      exit_code = 1
+      continue
+
+    apptest_result = "Succeeded"
+    for fixture in fixtures:
+      if not run_test([mojo_shell, apptest + " --gtest_filter=" + fixture]):
+        apptest_result = "Failed test(s) in " + apptest
+        exit_code = 1
+    print apptest_result
+
+  return exit_code
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))