OLD | NEW |
1 # Copyright 2007 Google Inc. Released under the GPL v2 | 1 # Copyright 2007 Google Inc. Released under the GPL v2 |
2 | 2 |
3 import re, os, sys, traceback, subprocess, time, pickle, glob, tempfile | 3 import re, os, sys, traceback, subprocess, time, pickle, glob, tempfile |
4 import logging, getpass | 4 import logging, getpass |
5 from autotest_lib.server import installable_object, prebuild, utils | 5 from autotest_lib.server import installable_object, prebuild, utils |
6 from autotest_lib.client.common_lib import base_job, log, error, autotemp | 6 from autotest_lib.client.common_lib import base_job, log, error, autotemp |
7 from autotest_lib.client.common_lib import global_config, packages | 7 from autotest_lib.client.common_lib import global_config, packages |
8 from autotest_lib.client.common_lib import utils as client_utils | 8 from autotest_lib.client.common_lib import utils as client_utils |
9 | 9 |
10 AUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' | 10 AUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' |
(...skipping 265 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
276 try: | 276 try: |
277 utils.system('tools/make_clean', ignore_status=True) | 277 utils.system('tools/make_clean', ignore_status=True) |
278 finally: | 278 finally: |
279 os.chdir(cwd) | 279 os.chdir(cwd) |
280 super(BaseAutotest, self).get(location) | 280 super(BaseAutotest, self).get(location) |
281 self.got = True | 281 self.got = True |
282 | 282 |
283 | 283 |
284 def run(self, control_file, results_dir='.', host=None, timeout=None, | 284 def run(self, control_file, results_dir='.', host=None, timeout=None, |
285 tag=None, parallel_flag=False, background=False, | 285 tag=None, parallel_flag=False, background=False, |
286 client_disconnect_timeout=1800): | 286 client_disconnect_timeout=1800, ignore_aborts=False): |
287 """ | 287 """ |
288 Run an autotest job on the remote machine. | 288 Run an autotest job on the remote machine. |
289 | 289 |
290 @param control_file: An open file-like-obj of the control file. | 290 @param control_file: An open file-like-obj of the control file. |
291 @param results_dir: A str path where the results should be stored | 291 @param results_dir: A str path where the results should be stored |
292 on the local filesystem. | 292 on the local filesystem. |
293 @param host: A Host instance on which the control file should | 293 @param host: A Host instance on which the control file should |
294 be run. | 294 be run. |
295 @param timeout: Maximum number of seconds to wait for the run or None. | 295 @param timeout: Maximum number of seconds to wait for the run or None. |
296 @param tag: Tag name for the client side instance of autotest. | 296 @param tag: Tag name for the client side instance of autotest. |
297 @param parallel_flag: Flag set when multiple jobs are run at the | 297 @param parallel_flag: Flag set when multiple jobs are run at the |
298 same time. | 298 same time. |
299 @param background: Indicates that the client should be launched as | 299 @param background: Indicates that the client should be launched as |
300 a background job; the code calling run will be responsible | 300 a background job; the code calling run will be responsible |
301 for monitoring the client and collecting the results. | 301 for monitoring the client and collecting the results. |
302 @param client_disconnect_timeout: Seconds to wait for the remote host | 302 @param client_disconnect_timeout: Seconds to wait for the remote host |
303 to come back after a reboot. [default: 30 minutes] | 303 to come back after a reboot. [default: 30 minutes] |
| 304 @param ignore_aborts: If the Autotest client aborts unexpectedly, don't |
| 305 record job status as ABORT. |
304 | 306 |
305 @raises AutotestRunError: If there is a problem executing | 307 @raises AutotestRunError: If there is a problem executing |
306 the control file. | 308 the control file. |
307 """ | 309 """ |
308 host = self._get_host_and_setup(host) | 310 host = self._get_host_and_setup(host) |
309 results_dir = os.path.abspath(results_dir) | 311 results_dir = os.path.abspath(results_dir) |
310 | 312 |
311 if tag: | 313 if tag: |
312 results_dir = os.path.join(results_dir, tag) | 314 results_dir = os.path.join(results_dir, tag) |
313 | 315 |
314 atrun = _Run(host, results_dir, tag, parallel_flag, background) | 316 atrun = _Run(host, results_dir, tag, parallel_flag, background) |
315 self._do_run(control_file, results_dir, host, atrun, timeout, | 317 self._do_run(control_file, results_dir, host, atrun, timeout, |
316 client_disconnect_timeout) | 318 client_disconnect_timeout, ignore_aborts) |
317 | 319 |
318 | 320 |
319 def _get_host_and_setup(self, host): | 321 def _get_host_and_setup(self, host): |
320 if not host: | 322 if not host: |
321 host = self.host | 323 host = self.host |
322 if not self.installed: | 324 if not self.installed: |
323 self.install(host) | 325 self.install(host) |
324 | 326 |
325 host.wait_up(timeout=30) | 327 host.wait_up(timeout=30) |
326 return host | 328 return host |
327 | 329 |
328 | 330 |
329 def _do_run(self, control_file, results_dir, host, atrun, timeout, | 331 def _do_run(self, control_file, results_dir, host, atrun, timeout, |
330 client_disconnect_timeout): | 332 client_disconnect_timeout, ignore_aborts): |
331 try: | 333 try: |
332 atrun.verify_machine() | 334 atrun.verify_machine() |
333 except: | 335 except: |
334 logging.error("Verify failed on %s. Reinstalling autotest", | 336 logging.error("Verify failed on %s. Reinstalling autotest", |
335 host.hostname) | 337 host.hostname) |
336 self.install(host) | 338 self.install(host) |
337 atrun.verify_machine() | 339 atrun.verify_machine() |
338 debug = os.path.join(results_dir, 'debug') | 340 debug = os.path.join(results_dir, 'debug') |
339 try: | 341 try: |
340 os.makedirs(debug) | 342 os.makedirs(debug) |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
389 host.send_file(state_file, atrun.remote_control_file + '.init.state') | 391 host.send_file(state_file, atrun.remote_control_file + '.init.state') |
390 os.remove(state_file) | 392 os.remove(state_file) |
391 | 393 |
392 # Copy control_file to remote_control_file on the host | 394 # Copy control_file to remote_control_file on the host |
393 host.send_file(tmppath, atrun.remote_control_file) | 395 host.send_file(tmppath, atrun.remote_control_file) |
394 if os.path.abspath(tmppath) != os.path.abspath(control_file): | 396 if os.path.abspath(tmppath) != os.path.abspath(control_file): |
395 os.remove(tmppath) | 397 os.remove(tmppath) |
396 | 398 |
397 atrun.execute_control( | 399 atrun.execute_control( |
398 timeout=timeout, | 400 timeout=timeout, |
399 client_disconnect_timeout=client_disconnect_timeout) | 401 client_disconnect_timeout=client_disconnect_timeout, |
| 402 ignore_aborts=ignore_aborts) |
400 | 403 |
401 | 404 |
402 def run_timed_test(self, test_name, results_dir='.', host=None, | 405 def run_timed_test(self, test_name, results_dir='.', host=None, |
403 timeout=None, *args, **dargs): | 406 timeout=None, *args, **dargs): |
404 """ | 407 """ |
405 Assemble a tiny little control file to just run one test, | 408 Assemble a tiny little control file to just run one test, |
406 and run it as an autotest client-side test | 409 and run it as an autotest client-side test |
407 """ | 410 """ |
408 if not host: | 411 if not host: |
409 host = self.host | 412 host = self.host |
(...skipping 243 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
653 return result | 656 return result |
654 elif not self.host.wait_up(client_disconnect_timeout): | 657 elif not self.host.wait_up(client_disconnect_timeout): |
655 raise error.AutoservSSHTimeout( | 658 raise error.AutoservSSHTimeout( |
656 "client was disconnected, reconnect timed out") | 659 "client was disconnected, reconnect timed out") |
657 finally: | 660 finally: |
658 client_log.close() | 661 client_log.close() |
659 self.host.job.pop_execution_context() | 662 self.host.job.pop_execution_context() |
660 | 663 |
661 | 664 |
662 def execute_section(self, section, timeout, stderr_redirector, | 665 def execute_section(self, section, timeout, stderr_redirector, |
663 client_disconnect_timeout): | 666 client_disconnect_timeout, ignore_aborts=False): |
664 logging.info("Executing %s/bin/autotest %s/control phase %d", | 667 logging.info("Executing %s/bin/autotest %s/control phase %d", |
665 self.autodir, self.autodir, section) | 668 self.autodir, self.autodir, section) |
666 | 669 |
667 if self.background: | 670 if self.background: |
668 result = self._execute_in_background(section, timeout) | 671 result = self._execute_in_background(section, timeout) |
669 else: | 672 else: |
670 result = self._execute_daemon(section, timeout, stderr_redirector, | 673 result = self._execute_daemon(section, timeout, stderr_redirector, |
671 client_disconnect_timeout) | 674 client_disconnect_timeout) |
672 | 675 |
673 last_line = stderr_redirector.last_line | 676 last_line = stderr_redirector.last_line |
674 | 677 |
675 # check if we failed hard enough to warrant an exception | 678 # check if we failed hard enough to warrant an exception |
676 if result.exit_status == 1: | 679 if result.exit_status == 1: |
677 err = error.AutotestRunError("client job was aborted") | 680 err = error.AutotestRunError("client job was aborted") |
678 elif not self.background and not result.stderr: | 681 elif not self.background and not result.stderr: |
679 err = error.AutotestRunError( | 682 err = error.AutotestRunError( |
680 "execute_section %s failed to return anything\n" | 683 "execute_section %s failed to return anything\n" |
681 "stdout:%s\n" % (section, result.stdout)) | 684 "stdout:%s\n" % (section, result.stdout)) |
682 else: | 685 else: |
683 err = None | 686 err = None |
684 | 687 |
685 # log something if the client failed AND never finished logging | 688 # log something if the client failed AND never finished logging |
686 if err and not self.is_client_job_finished(last_line): | 689 if (err and not self.is_client_job_finished(last_line) |
| 690 and not ignore_aborts): |
687 self.log_unexpected_abort(stderr_redirector) | 691 self.log_unexpected_abort(stderr_redirector) |
688 | 692 |
689 if err: | 693 if err: |
690 raise err | 694 raise err |
691 else: | 695 else: |
692 return stderr_redirector.last_line | 696 return stderr_redirector.last_line |
693 | 697 |
694 | 698 |
695 def _wait_for_reboot(self, old_boot_id): | 699 def _wait_for_reboot(self, old_boot_id): |
696 logging.info("Client is rebooting") | 700 logging.info("Client is rebooting") |
(...skipping 14 matching lines...) Expand all Loading... |
711 self.host.hardreset(wait=False) | 715 self.host.hardreset(wait=False) |
712 except (AttributeError, error.AutoservUnsupportedError): | 716 except (AttributeError, error.AutoservUnsupportedError): |
713 warning = "Hard reset unsupported on %s" | 717 warning = "Hard reset unsupported on %s" |
714 warning %= self.host.hostname | 718 warning %= self.host.hostname |
715 logging.warning(warning) | 719 logging.warning(warning) |
716 raise error.AutotestRunError("%s failed to boot after %ds" % | 720 raise error.AutotestRunError("%s failed to boot after %ds" % |
717 (self.host.hostname, BOOT_TIME)) | 721 (self.host.hostname, BOOT_TIME)) |
718 self.host.reboot_followup() | 722 self.host.reboot_followup() |
719 | 723 |
720 | 724 |
721 def execute_control(self, timeout=None, client_disconnect_timeout=None): | 725 def execute_control(self, timeout=None, client_disconnect_timeout=None, |
| 726 ignore_aborts=False): |
722 if not self.background: | 727 if not self.background: |
723 collector = log_collector(self.host, self.tag, self.results_dir) | 728 collector = log_collector(self.host, self.tag, self.results_dir) |
724 hostname = self.host.hostname | 729 hostname = self.host.hostname |
725 remote_results = collector.client_results_dir | 730 remote_results = collector.client_results_dir |
726 local_results = collector.server_results_dir | 731 local_results = collector.server_results_dir |
727 self.host.job.add_client_log(hostname, remote_results, | 732 self.host.job.add_client_log(hostname, remote_results, |
728 local_results) | 733 local_results) |
729 job_record_context = self.host.job.get_record_context() | 734 job_record_context = self.host.job.get_record_context() |
730 | 735 |
731 section = 0 | 736 section = 0 |
732 start_time = time.time() | 737 start_time = time.time() |
733 | 738 |
734 logger = client_logger(self.host, self.tag, self.results_dir) | 739 logger = client_logger(self.host, self.tag, self.results_dir) |
735 try: | 740 try: |
736 while not timeout or time.time() < start_time + timeout: | 741 while not timeout or time.time() < start_time + timeout: |
737 if timeout: | 742 if timeout: |
738 section_timeout = start_time + timeout - time.time() | 743 section_timeout = start_time + timeout - time.time() |
739 else: | 744 else: |
740 section_timeout = None | 745 section_timeout = None |
741 boot_id = self.host.get_boot_id() | 746 boot_id = self.host.get_boot_id() |
742 last = self.execute_section(section, section_timeout, | 747 last = self.execute_section(section, section_timeout, |
743 logger, client_disconnect_timeout) | 748 logger, client_disconnect_timeout, |
| 749 ignore_aborts) |
744 if self.background: | 750 if self.background: |
745 return | 751 return |
746 section += 1 | 752 section += 1 |
747 if self.is_client_job_finished(last): | 753 if self.is_client_job_finished(last): |
748 logging.info("Client complete") | 754 logging.info("Client complete") |
749 return | 755 return |
750 elif self.is_client_job_rebooting(last): | 756 elif self.is_client_job_rebooting(last): |
751 try: | 757 try: |
752 self._wait_for_reboot(boot_id) | 758 self._wait_for_reboot(boot_id) |
753 except error.AutotestRunError, e: | 759 except error.AutotestRunError, e: |
(...skipping 312 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1066 | 1072 |
1067 @returns: True if the test passes, False otherwise.""" | 1073 @returns: True if the test passes, False otherwise.""" |
1068 at = self._Autotest() | 1074 at = self._Autotest() |
1069 control_file = ('result = job.run_test(%s)\n' | 1075 control_file = ('result = job.run_test(%s)\n' |
1070 'job.set_state("test_result", result)\n') | 1076 'job.set_state("test_result", result)\n') |
1071 test_args = [repr(test_name)] | 1077 test_args = [repr(test_name)] |
1072 test_args += ['%s=%r' % (k, v) for k, v in dargs.iteritems()] | 1078 test_args += ['%s=%r' % (k, v) for k, v in dargs.iteritems()] |
1073 control_file %= ', '.join(test_args) | 1079 control_file %= ', '.join(test_args) |
1074 at.run(control_file, host=self) | 1080 at.run(control_file, host=self) |
1075 return at.job.get_state('test_result', default=False) | 1081 return at.job.get_state('test_result', default=False) |
OLD | NEW |