OLD | NEW |
1 """Database model classes for the scheduler. | 1 """Database model classes for the scheduler. |
2 | 2 |
3 Contains model classes abstracting the various DB tables used by the scheduler. | 3 Contains model classes abstracting the various DB tables used by the scheduler. |
4 These overlap the Django models in basic functionality, but were written before | 4 These overlap the Django models in basic functionality, but were written before |
5 the Django models existed and have not yet been phased out. Some of them | 5 the Django models existed and have not yet been phased out. Some of them |
6 (particularly HostQueueEntry and Job) have considerable scheduler-specific logic | 6 (particularly HostQueueEntry and Job) have considerable scheduler-specific logic |
7 which would probably be ill-suited for inclusion in the general Django model | 7 which would probably be ill-suited for inclusion in the general Django model |
8 classes. | 8 classes. |
9 | 9 |
10 Globals: | 10 Globals: |
11 _notify_email_statuses: list of HQE statuses. each time a single HQE reaches | 11 _notify_email_statuses: list of HQE statuses. each time a single HQE reaches |
12 one of these statuses, an email will be sent to the job's email_list. | 12 one of these statuses, an email will be sent to the job's email_list. |
13 comes from global_config. | 13 comes from global_config. |
14 _base_url: URL to the local AFE server, used to construct URLs for emails. | 14 _base_url: URL to the local AFE server, used to construct URLs for emails. |
15 _db: DatabaseConnection for this module. | 15 _db: DatabaseConnection for this module. |
16 _drone_manager: reference to global DroneManager instance. | 16 _drone_manager: reference to global DroneManager instance. |
17 """ | 17 """ |
18 | 18 |
19 import datetime, itertools, logging, os, re, sys, time, weakref | 19 import datetime, itertools, logging, os, re, sys, time, weakref |
20 from django.db import connection | 20 from django.db import connection |
21 from autotest_lib.client.common_lib import global_config, host_protections | 21 from autotest_lib.client.common_lib import global_config, host_protections |
| 22 from autotest_lib.client.common_lib import global_config, utils |
22 from autotest_lib.frontend.afe import models, model_attributes | 23 from autotest_lib.frontend.afe import models, model_attributes |
23 from autotest_lib.database import database_connection | 24 from autotest_lib.database import database_connection |
24 from autotest_lib.scheduler import drone_manager, email_manager | 25 from autotest_lib.scheduler import drone_manager, email_manager |
25 from autotest_lib.scheduler import scheduler_config | 26 from autotest_lib.scheduler import scheduler_config |
26 | 27 |
27 _notify_email_statuses = [] | 28 _notify_email_statuses = [] |
28 _base_url = None | 29 _base_url = None |
29 | 30 |
30 _db = None | 31 _db = None |
31 _drone_manager = None | 32 _drone_manager = None |
(...skipping 552 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
584 | 585 |
585 if not self.execution_subdir: | 586 if not self.execution_subdir: |
586 return | 587 return |
587 # unregister any possible pidfiles associated with this queue entry | 588 # unregister any possible pidfiles associated with this queue entry |
588 for pidfile_name in drone_manager.ALL_PIDFILE_NAMES: | 589 for pidfile_name in drone_manager.ALL_PIDFILE_NAMES: |
589 pidfile_id = _drone_manager.get_pidfile_id_from( | 590 pidfile_id = _drone_manager.get_pidfile_id_from( |
590 self.execution_path(), pidfile_name=pidfile_name) | 591 self.execution_path(), pidfile_name=pidfile_name) |
591 _drone_manager.unregister_pidfile(pidfile_id) | 592 _drone_manager.unregister_pidfile(pidfile_id) |
592 | 593 |
593 | 594 |
| 595 def _get_status_email_contents(self, status, summary=None, hostname=None): |
| 596 """ |
| 597 Gather info for the status notification e-mails. |
| 598 |
| 599 If needed, we could start using the Django templating engine to create |
| 600 the subject and the e-mail body, but that doesn't seem necessary right |
| 601 now. |
| 602 |
| 603 @param status: Job status text. Mandatory. |
| 604 @param summary: Job summary text. Optional. |
| 605 @param hostname: A hostname for the job. Optional. |
| 606 |
| 607 @return: Tuple (subject, body) for the notification e-mail. |
| 608 """ |
| 609 job_stats = Job(id=self.job.id).get_execution_details() |
| 610 |
| 611 subject = ('Autotest | Job ID: %s "%s" | Status: %s ' % |
| 612 (self.job.id, self.job.name, status)) |
| 613 |
| 614 if hostname is not None: |
| 615 subject += '| Hostname: %s ' % hostname |
| 616 |
| 617 if status not in ["1 Failed", "Failed"]: |
| 618 subject += '| Success Rate: %.2f %%' % job_stats['success_rate'] |
| 619 |
| 620 body = "Job ID: %s\n" % self.job.id |
| 621 body += "Job name: %s\n" % self.job.name |
| 622 if hostname is not None: |
| 623 body += "Host: %s\n" % hostname |
| 624 if summary is not None: |
| 625 body += "Summary: %s\n" % summary |
| 626 body += "Status: %s\n" % status |
| 627 body += "Results interface URL: %s\n" % self._view_job_url() |
| 628 body += "Execution time (HH:MM:SS): %s\n" % job_stats['execution_time'] |
| 629 if int(job_stats['total_executed']) > 0: |
| 630 body += "User tests executed: %s\n" % job_stats['total_executed'] |
| 631 body += "User tests passed: %s\n" % job_stats['total_passed'] |
| 632 body += "User tests failed: %s\n" % job_stats['total_failed'] |
| 633 body += ("User tests success rate: %.2f %%\n" % |
| 634 job_stats['success_rate']) |
| 635 |
| 636 if job_stats['failed_rows']: |
| 637 body += "Failures:\n" |
| 638 body += job_stats['failed_rows'] |
| 639 |
| 640 return subject, body |
| 641 |
| 642 |
594 def _email_on_status(self, status): | 643 def _email_on_status(self, status): |
595 hostname = self._get_hostname() | 644 hostname = self._get_hostname() |
596 | 645 subject, body = self._get_status_email_contents(status, None, hostname) |
597 subject = 'Autotest: Job ID: %s "%s" Host: %s %s' % ( | |
598 self.job.id, self.job.name, hostname, status) | |
599 body = "Job ID: %s\nJob Name: %s\nHost: %s\nStatus: %s\n%s\n" % ( | |
600 self.job.id, self.job.name, hostname, status, | |
601 self._view_job_url()) | |
602 email_manager.manager.send_email(self.job.email_list, subject, body) | 646 email_manager.manager.send_email(self.job.email_list, subject, body) |
603 | 647 |
604 | 648 |
605 def _email_on_job_complete(self): | 649 def _email_on_job_complete(self): |
606 if not self.job.is_finished(): | 650 if not self.job.is_finished(): |
607 return | 651 return |
608 | 652 |
609 summary_text = [] | 653 summary = [] |
610 hosts_queue = HostQueueEntry.fetch('job_id = %s' % self.job.id) | 654 hosts_queue = HostQueueEntry.fetch('job_id = %s' % self.job.id) |
611 for queue_entry in hosts_queue: | 655 for queue_entry in hosts_queue: |
612 summary_text.append("Host: %s Status: %s" % | 656 summary.append("Host: %s Status: %s" % |
613 (queue_entry._get_hostname(), | 657 (queue_entry._get_hostname(), |
614 queue_entry.status)) | 658 queue_entry.status)) |
615 | 659 |
616 summary_text = "\n".join(summary_text) | 660 summary = "\n".join(summary) |
617 status_counts = models.Job.objects.get_status_counts( | 661 status_counts = models.Job.objects.get_status_counts( |
618 [self.job.id])[self.job.id] | 662 [self.job.id])[self.job.id] |
619 status = ', '.join('%d %s' % (count, status) for status, count | 663 status = ', '.join('%d %s' % (count, status) for status, count |
620 in status_counts.iteritems()) | 664 in status_counts.iteritems()) |
621 | 665 |
622 subject = 'Autotest: Job ID: %s "%s" %s' % ( | 666 subject, body = self._get_status_email_contents(status, summary, None) |
623 self.job.id, self.job.name, status) | |
624 body = "Job ID: %s\nJob Name: %s\nStatus: %s\n%s\nSummary:\n%s" % ( | |
625 self.job.id, self.job.name, status, self._view_job_url(), | |
626 summary_text) | |
627 email_manager.manager.send_email(self.job.email_list, subject, body) | 667 email_manager.manager.send_email(self.job.email_list, subject, body) |
628 | 668 |
629 | 669 |
630 def schedule_pre_job_tasks(self): | 670 def schedule_pre_job_tasks(self): |
631 logging.info("%s/%s/%s (job %s, entry %s) scheduled on %s, status=%s", | 671 logging.info("%s/%s/%s (job %s, entry %s) scheduled on %s, status=%s", |
632 self.job.name, self.meta_host, self.atomic_group_id, | 672 self.job.name, self.meta_host, self.atomic_group_id, |
633 self.job.id, self.id, self.host.hostname, self.status) | 673 self.job.id, self.id, self.host.hostname, self.status) |
634 | 674 |
635 self._do_schedule_pre_job_tasks() | 675 self._do_schedule_pre_job_tasks() |
636 | 676 |
(...skipping 177 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
814 SELECT * FROM afe_host_queue_entries | 854 SELECT * FROM afe_host_queue_entries |
815 WHERE job_id= %s | 855 WHERE job_id= %s |
816 """, (self.id,)) | 856 """, (self.id,)) |
817 entries = [HostQueueEntry(row=i) for i in rows] | 857 entries = [HostQueueEntry(row=i) for i in rows] |
818 | 858 |
819 assert len(entries)>0 | 859 assert len(entries)>0 |
820 | 860 |
821 return entries | 861 return entries |
822 | 862 |
823 | 863 |
| 864 def get_execution_details(self): |
| 865 """ |
| 866 Get test execution details for this job. |
| 867 |
| 868 @return: Dictionary with test execution details |
| 869 """ |
| 870 def _find_test_jobs(rows): |
| 871 """ |
| 872 Here we are looking for tests such as SERVER_JOB and CLIENT_JOB.* |
| 873 Those are autotest 'internal job' tests, so they should not be |
| 874 counted when evaluating the test stats. |
| 875 |
| 876 @param rows: List of rows (matrix) with database results. |
| 877 """ |
| 878 job_test_pattern = re.compile('SERVER|CLIENT\\_JOB\.[\d]') |
| 879 n_test_jobs = 0 |
| 880 for r in rows: |
| 881 test_name = r[0] |
| 882 if job_test_pattern.match(test_name): |
| 883 n_test_jobs += 1 |
| 884 |
| 885 return n_test_jobs |
| 886 |
| 887 stats = {} |
| 888 |
| 889 rows = _db.execute(""" |
| 890 SELECT t.test, s.word, t.reason |
| 891 FROM tko_tests AS t, tko_jobs AS j, tko_status AS s |
| 892 WHERE t.job_idx = j.job_idx |
| 893 AND s.status_idx = t.status |
| 894 AND j.afe_job_id = %s |
| 895 """ % self.id) |
| 896 |
| 897 failed_rows = [r for r in rows if not 'GOOD' in r] |
| 898 |
| 899 n_test_jobs = _find_test_jobs(rows) |
| 900 n_test_jobs_failed = _find_test_jobs(failed_rows) |
| 901 |
| 902 total_executed = len(rows) - n_test_jobs |
| 903 total_failed = len(failed_rows) - n_test_jobs_failed |
| 904 |
| 905 if total_executed > 0: |
| 906 success_rate = 100 - ((total_failed / float(total_executed)) * 100) |
| 907 else: |
| 908 success_rate = 0 |
| 909 |
| 910 stats['total_executed'] = total_executed |
| 911 stats['total_failed'] = total_failed |
| 912 stats['total_passed'] = total_executed - total_failed |
| 913 stats['success_rate'] = success_rate |
| 914 |
| 915 status_header = ("Test Name", "Status", "Reason") |
| 916 if failed_rows: |
| 917 stats['failed_rows'] = utils.matrix_to_string(failed_rows, |
| 918 status_header) |
| 919 else: |
| 920 stats['failed_rows'] = '' |
| 921 |
| 922 time_row = _db.execute(""" |
| 923 SELECT started_time, finished_time |
| 924 FROM tko_jobs |
| 925 WHERE afe_job_id = %s |
| 926 """ % self.id) |
| 927 |
| 928 if time_row: |
| 929 t_begin, t_end = time_row[0] |
| 930 delta = t_end - t_begin |
| 931 minutes, seconds = divmod(delta.seconds, 60) |
| 932 hours, minutes = divmod(minutes, 60) |
| 933 stats['execution_time'] = ("%02d:%02d:%02d" % |
| 934 (hours, minutes, seconds)) |
| 935 else: |
| 936 stats['execution_time'] = '(none)' |
| 937 |
| 938 return stats |
| 939 |
| 940 |
824 def set_status(self, status, update_queues=False): | 941 def set_status(self, status, update_queues=False): |
825 self.update_field('status',status) | 942 self.update_field('status',status) |
826 | 943 |
827 if update_queues: | 944 if update_queues: |
828 for queue_entry in self.get_host_queue_entries(): | 945 for queue_entry in self.get_host_queue_entries(): |
829 queue_entry.set_status(status) | 946 queue_entry.set_status(status) |
830 | 947 |
831 | 948 |
832 def keyval_dict(self): | 949 def keyval_dict(self): |
833 return self.model().keyval_dict() | 950 return self.model().keyval_dict() |
(...skipping 353 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1187 def abort_delay_ready_task(self): | 1304 def abort_delay_ready_task(self): |
1188 """Abort the delayed task associated with this job, if any.""" | 1305 """Abort the delayed task associated with this job, if any.""" |
1189 if self._delay_ready_task: | 1306 if self._delay_ready_task: |
1190 # Cancel any pending callback that would try to run again | 1307 # Cancel any pending callback that would try to run again |
1191 # as we are already running. | 1308 # as we are already running. |
1192 self._delay_ready_task.abort() | 1309 self._delay_ready_task.abort() |
1193 | 1310 |
1194 | 1311 |
1195 def __str__(self): | 1312 def __str__(self): |
1196 return '%s-%s' % (self.id, self.owner) | 1313 return '%s-%s' % (self.id, self.owner) |
OLD | NEW |