autotest/factory: Interrupt tests by factory UI.

client/site_tests/suite_Factory/control

 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 AUTHOR = "Chrome OS Team"
 NAME = "Factory"
 TIME = "LONG"
 TEST_CATEGORY = "Functional"
 TEST_CLASS = "suite"
 TEST_TYPE = "client"
 
 DOC = """
 This suite executed all of the factory tests, and incorporates an
 X-windows GTK-based UI to highlight testing results and to allow
 factory operators to switch between tests, and to re-run tests, all
 on-demand via keyboard shortcuts.
 
 The UI is implemented as a seperate process (see _FACTORY_UI_PATH),
 which means that interprocess communication is needed between this
 control, the UI, and the tests (which are forked children of this
 control process).  """
 
 
 import imp
 import os
 import pprint
+import signal
+import traceback
 
 imp.load_source('common', job.autodir + '/bin/common.py')
 
 from autotest_lib.client.bin import factory
 from autotest_lib.client.bin import parallel
 
 
 FACTORY_UI_PATH = job.autodir + '/bin/factory_ui'
 FACTORY_STATE_SERVER_PATH = job.autodir + '/bin/factory_state.py'
 STATUS_FILE_PATH = job.autodir + '/results/default/status'
 TEST_LIST_PATH = job.autodir + '/site_tests/suite_Factory/test_list'
 
 
 # Hack to grab the pid for forked tests.
 
 from autotest_lib.client.bin.parallel import fork_waitfor as orig_fork_waitfor
 
-def new_fork_waitfor(tmp, pid):
+def factory_fork_waitfor(tmp, pid):
     factory.set_shared_data('active_test_data', (tmp, pid))
-    try:
-        orig_fork_waitfor(tmp, pid)
-    except os.error, e:
-        raise error.TestError('Test Interrupted (maybe keyboard shortcut)')
+    (pid, status) = os.waitpid(pid, 0)
+    parallel._check_for_subprocess_exception(tmp, pid)
+    if status == signal.SIGTERM:
+        raise error.TestError('Test TERMINATED (return code = %s),\n'
+                              'Maybe caused by hitting keyboard shotcut.' %
+                              status)
+    elif status:
+        raise error.TestError("Test FAILED with return code: %d" % (status))
 
-parallel.fork_waitfor = new_fork_waitfor
+parallel.fork_waitfor = factory_fork_waitfor
 
 
 # These definitions are expose these classes directly into this
 # namespace, so that the following exec'ed file can have cleaner
 # syntax.  These are done in this fashion, as opposed to "from factory
 # import <class>" to work-around Python namespace wackiness -- the
 # from syntax does not work, creating new classes.
 OperatorTest = factory.OperatorTest
 InformationScreen = factory.InformationScreen
 AutomatedSequence = factory.AutomatedSequence
@@ skipping 74 matching lines @@
 
         def notify(self, test, status):
             self.notify_fd.write(' ')
             self.notify_fd.flush()
     try:
         state_server_process = start_state_server()
         ui_process = start_ui()
         ui_notify_fd = ui_process.stdin
         ui_notify_obj = FactoryUINotifyObject(ui_notify_fd)
         main_loop(next_subtest_info, ui_notify_obj.notify)
+    except:
+        factory.log('suite_Factory failed with exception: ' +
+                    traceback.format_exc())
+        # To fix a broken status tree, force a step_init here.
+        job.next_step([step_init])
     finally:
         stop_process(ui_process, 'ui')
         stop_process(state_server_process, 'state server')
 
 def main_loop(next_subtest_info, notify_callback):
     test_db = factory.TestDatabase(TEST_LIST)
     status_map = factory.StatusMap(TEST_LIST, STATUS_FILE_PATH,
                                    status_change_callback=notify_callback)
 
     # initialize factory test states
@@ skipping 44 matching lines @@
 
     test = find_next_test_to_run(status_map, test_db)
     while test is not None:
         unique_id_str = test_db.get_unique_id_str(test)
         factory.log('control running test %s' % unique_id_str)
         if isinstance(test, factory.AutomatedSequence):
             shortcut_target = run_automated_sequence(test)
         else:
             shortcut_target = control_state.run_test(test)
         test = shortcut_target or find_next_test_to_run(status_map, test_db)