Fixes for cbuild to work with ctest.

bin/cros_au_test_harness.py

Index: bin/cros_au_test_harness.py
diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py
index a495f06f9bd21230ec05da69933b70ad716b9c9c..799a5240c878f8d314df4ffbc1f74cb9924b4adb 100755
--- a/bin/cros_au_test_harness.py
+++ b/bin/cros_au_test_harness.py
@@ -10,7 +10,12 @@ import sys
 import unittest
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
-from cros_build_lib import RunCommand, Info, Warning, ReinterpretPathForChroot
+from cros_build_lib import Die
+from cros_build_lib import Info
+from cros_build_lib import ReinterpretPathForChroot
+from cros_build_lib import RunCommand
+from cros_build_lib import Warning
+
 
 _KVM_PID_FILE = '/tmp/harness_pid'
 _FULL_VDISK_SIZE = 6072
@@ -21,6 +26,8 @@ global base_image_path
 global board
 global remote
 global target_image_path
+global vm_graphics_flag
+
 
 _VERIFY_SUITE = 'suite_Smoke'
 
@@ -41,6 +48,20 @@ class AUTest(object):
 
     return stateful_change_flag
 
+  def ParseGenerateTestReportOutput(self, output):
+    """Returns the percentage of tests that passed based on output."""
+    percent_passed = 0
+    lines = output.split('\n')
+
+    for line in lines:
+      if line.startswith("Total PASS:"):
+        # FORMAT: ^TOTAL PASS: num_passed/num_total (percent%)$
+        percent_passed = line.split()[3].strip('()%')
+        Info('Percent of tests passed %s' % percent_passed)
+        break
+
+    return int(percent_passed)
+
   def PrepareBase(self):
     """Prepares target with base_image_path."""
     pass
@@ -58,51 +79,90 @@ class AUTest(object):
     """
     pass
 
-  def VerifyImage(self):
-    """Verifies the image is correct."""
+  def VerifyImage(self, percent_required_to_pass):
+    """Verifies the image with tests.
+
+    Verifies that the test images passes the percent required.
+
+    Args:
+      percent_required_to_pass:  percentage required to pass.  This should be
+        fall between 0-100.
+
+    Returns:
+      Returns the percent that passed.
+    """
     pass
 
+  def CommonVerifyImage(self, unittest, output, percent_required_to_pass):
+    """Helper function for VerifyImage that returns percent of tests passed.
+
+    Takes output from a test suite, verifies the number of tests passed is
+    sufficient and outputs info.
+
+    Args:
+      unittest: Handle to the unittest.
+      output: stdout from a test run.
+      percent_required_to_pass: percentage required to pass.  This should be
+        fall between 0-100.
+    Returns:
+      percent that passed.
+    """
+    Info('Output from VerifyImage():')
+    print output
+    percent_passed = self.ParseGenerateTestReportOutput(output)
+    Info('Percent passed: %d vs. Percent required: %d' % (
+        percent_passed, percent_required_to_pass))
+    unittest.assertTrue(percent_passed >=
+                        percent_required_to_pass)
+    return percent_passed
+
   def testFullUpdateKeepStateful(self):
     """Tests if we can update normally.
 
     This test checks that we can update by updating the stateful partition
     rather than wiping it.
     """
-    # Prepare and verify the base image has been prepared correctly.
+    # Just make sure some tests pass on original image.  Some old images
+    # don't pass many tests.
     self.PrepareBase()
-    self.VerifyImage()
+    # TODO(sosa): move to 100% once we start testing using the autotest paired
+    # with the dev channel.
+    percent_passed = self.VerifyImage(42)
 
-    # Update to.
+    # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image.')
     self.UpdateImage(target_image_path)
-    self.VerifyImage()
+    self.VerifyImage(100)
 
-    # Update from.
+    # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image on vm back to base image.')
     self.UpdateImage(base_image_path)
-    self.VerifyImage()
+    self.VerifyImage(percent_passed)
 
   # TODO(sosa): Re-enable once we have a good way of checking for version
-  # compatability.
+  # compatibility.
   def NotestFullUpdateWipeStateful(self):
     """Tests if we can update after cleaning the stateful partition.
 
     This test checks that we can update successfully after wiping the
     stateful partition.
     """
-    # Prepare and verify the base image has been prepared correctly.
+    # Just make sure some tests pass on original image.  Some old images
+    # don't pass many tests.
     self.PrepareBase()
-    self.VerifyImage()
+    # TODO(sosa): move to 100% once we start testing using the autotest paired
+    # with the dev channel.
+    percent_passed = self.VerifyImage(42)
 
-    # Update to.
+    # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image and wiping stateful.')
     self.UpdateImage(target_image_path, 'clean')
-    self.VerifyImage()
+    self.VerifyImage(100)
 
-    # Update from.
+    # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image back to base image and wiping stateful.')
     self.UpdateImage(base_image_path, 'clean')
-    self.VerifyImage()
+    self.VerifyImage(percent_passed)
 
 
 class RealAUTest(unittest.TestCase, AUTest):
@@ -111,6 +171,10 @@ class RealAUTest(unittest.TestCase, AUTest):
   def setUp(self):
     AUTest.setUp(self)
 
+  def PrepareBase(self):
+    """Auto-update to base image to prepare for test."""
+    self.UpdateImage(base_image_path)
+
   def UpdateImage(self, image_path, stateful_change='old'):
     """Updates a remote image using image_to_live.sh."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
@@ -124,13 +188,14 @@ class RealAUTest(unittest.TestCase, AUTest):
         ], enter_chroot=False)
 
 
-  def VerifyImage(self):
+  def VerifyImage(self, percent_required_to_pass):
     """Verifies an image using run_remote_tests.sh with verification suite."""
-    RunCommand([
+    output = RunCommand([
         '%s/run_remote_tests.sh' % self.crosutils,
         '--remote=%s' % remote,
         _VERIFY_SUITE,
-       ], error_ok=False, enter_chroot=False)
+       ], error_ok=True, enter_chroot=False, redirect_stdout=True)
+    return self.CommonVerifyImage(self, output, percent_required_to_pass)
 
 
 class VirtualAUTest(unittest.TestCase, AUTest):
@@ -181,22 +246,26 @@ class VirtualAUTest(unittest.TestCase, AUTest):
                 '--update_image_path=%s' % image_path,
                 '--vm_image_path=%s' % self.vm_image_path,
                 '--snapshot',
+                vm_graphics_flag,
                 '--persist',
                 '--kvm_pid=%s' % _KVM_PID_FILE,
                 stateful_change_flag,
                ], enter_chroot=False)
 
-  def VerifyImage(self):
+  def VerifyImage(self, percent_required_to_pass):
     """Runs vm smoke suite to verify image."""
     # image_to_live already verifies lsb-release matching.  This is just
     # for additional steps.
-    RunCommand(['%s/cros_run_vm_test' % self.crosutilsbin,
-                '--image_path=%s' % self.vm_image_path,
-                '--snapshot',
-                '--persist',
-                '--kvm_pid=%s' % _KVM_PID_FILE,
-                '--test_case=%s' % _VERIFY_SUITE,
-               ], error_ok=False, enter_chroot=False)
+    output = RunCommand(['%s/cros_run_vm_test' % self.crosutilsbin,
+                         '--image_path=%s' % self.vm_image_path,
+                         '--snapshot',
+                         '--persist',
+                         vm_graphics_flag,
+                         '--kvm_pid=%s' % _KVM_PID_FILE,
+                         '--test_case=%s' % _VERIFY_SUITE,
+                         ], error_ok=True, enter_chroot=False,
+                            redirect_stdout=True)
+    return self.CommonVerifyImage(self, output, percent_required_to_pass)
 
 
 if __name__ == '__main__':
@@ -211,6 +280,8 @@ if __name__ == '__main__':
                     help='type of test to run: [vm, real]. Default: vm.')
   parser.add_option('-m', '--remote',
                     help='Remote address for real test.')
+  parser.add_option('--no_graphics', action='store_true',
+                    help='Disable graphics for the vm test.')
   # Set the usage to include flags.
   parser.set_usage(parser.format_help())
   # Parse existing sys.argv so we can pass rest to unittest.main.
@@ -222,19 +293,24 @@ if __name__ == '__main__':
 
   if not base_image_path:
     parser.error('Need path to base image for vm.')
+  elif not os.path.exists(base_image_path):
+    Die('%s does not exist' % base_image_path)
 
   if not target_image_path:
     parser.error('Need path to target image to update with.')
+  elif not os.path.exists(target_image_path):
+    Die('%s does not exist' % target_image_path)
 
   if not board:
     parser.error('Need board to convert base image to vm.')
 
-  return_code = 0
+  vm_graphics_flag = ''
+  if options.no_graphics: vm_graphics_flag = '--no_graphics'
 
   # Only run the test harness we care about.
   if options.type == 'vm':
     suite = unittest.TestLoader().loadTestsFromTestCase(VirtualAUTest)
-    return_code = unittest.TextTestRunner(verbosity=2).run(suite)
+    test_result = unittest.TextTestRunner(verbosity=2).run(suite)
   elif options.type == 'real':
     if not options.remote:
       parser.error('Real tests require a remote test machine.')
@@ -242,8 +318,9 @@ if __name__ == '__main__':
       remote = options.remote
 
     suite = unittest.TestLoader().loadTestsFromTestCase(RealAUTest)
-    return_code = unittest.TextTestRunner(verbosity=2).run(suite)
+    test_result = unittest.TextTestRunner(verbosity=2).run(suite)
   else:
     parser.error('Could not parse harness type %s.' % options.type)
 
-  sys.exit(return_code)
+  if not test_result.wasSuccessful():
+    Die('Test harness was not successful')