OLD | NEW |
1 """Database model classes for the scheduler. | 1 """Database model classes for the scheduler. |
2 | 2 |
3 Contains model classes abstracting the various DB tables used by the scheduler. | 3 Contains model classes abstracting the various DB tables used by the scheduler. |
4 These overlap the Django models in basic functionality, but were written before | 4 These overlap the Django models in basic functionality, but were written before |
5 the Django models existed and have not yet been phased out. Some of them | 5 the Django models existed and have not yet been phased out. Some of them |
6 (particularly HostQueueEntry and Job) have considerable scheduler-specific logic | 6 (particularly HostQueueEntry and Job) have considerable scheduler-specific logic |
7 which would probably be ill-suited for inclusion in the general Django model | 7 which would probably be ill-suited for inclusion in the general Django model |
8 classes. | 8 classes. |
9 | 9 |
10 Globals: | 10 Globals: |
11 _notify_email_statuses: list of HQE statuses. each time a single HQE reaches | 11 _notify_email_statuses: list of HQE statuses. each time a single HQE reaches |
12 one of these statuses, an email will be sent to the job's email_list. | 12 one of these statuses, an email will be sent to the job's email_list. |
13 comes from global_config. | 13 comes from global_config. |
14 _base_url: URL to the local AFE server, used to construct URLs for emails. | 14 _base_url: URL to the local AFE server, used to construct URLs for emails. |
15 _db: DatabaseConnection for this module. | 15 _db: DatabaseConnection for this module. |
16 _drone_manager: reference to global DroneManager instance. | 16 _drone_manager: reference to global DroneManager instance. |
17 """ | 17 """ |
18 | 18 |
19 import datetime, itertools, logging, os, re, sys, time, weakref | 19 import datetime, itertools, logging, os, re, sys, time, weakref |
20 from django.db import connection | 20 from django.db import connection |
21 from autotest_lib.client.common_lib import global_config, host_protections | 21 from autotest_lib.client.common_lib import global_config, host_protections |
22 from autotest_lib.client.common_lib import global_config, utils | |
23 from autotest_lib.frontend.afe import models, model_attributes | 22 from autotest_lib.frontend.afe import models, model_attributes |
24 from autotest_lib.database import database_connection | 23 from autotest_lib.database import database_connection |
25 from autotest_lib.scheduler import drone_manager, email_manager | 24 from autotest_lib.scheduler import drone_manager, email_manager |
26 from autotest_lib.scheduler import scheduler_config | 25 from autotest_lib.scheduler import scheduler_config |
27 | 26 |
28 _notify_email_statuses = [] | 27 _notify_email_statuses = [] |
29 _base_url = None | 28 _base_url = None |
30 | 29 |
31 _db = None | 30 _db = None |
32 _drone_manager = None | 31 _drone_manager = None |
(...skipping 552 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
585 | 584 |
586 if not self.execution_subdir: | 585 if not self.execution_subdir: |
587 return | 586 return |
588 # unregister any possible pidfiles associated with this queue entry | 587 # unregister any possible pidfiles associated with this queue entry |
589 for pidfile_name in drone_manager.ALL_PIDFILE_NAMES: | 588 for pidfile_name in drone_manager.ALL_PIDFILE_NAMES: |
590 pidfile_id = _drone_manager.get_pidfile_id_from( | 589 pidfile_id = _drone_manager.get_pidfile_id_from( |
591 self.execution_path(), pidfile_name=pidfile_name) | 590 self.execution_path(), pidfile_name=pidfile_name) |
592 _drone_manager.unregister_pidfile(pidfile_id) | 591 _drone_manager.unregister_pidfile(pidfile_id) |
593 | 592 |
594 | 593 |
595 def _get_status_email_contents(self, status, summary=None, hostname=None): | |
596 """ | |
597 Gather info for the status notification e-mails. | |
598 | |
599 If needed, we could start using the Django templating engine to create | |
600 the subject and the e-mail body, but that doesn't seem necessary right | |
601 now. | |
602 | |
603 @param status: Job status text. Mandatory. | |
604 @param summary: Job summary text. Optional. | |
605 @param hostname: A hostname for the job. Optional. | |
606 | |
607 @return: Tuple (subject, body) for the notification e-mail. | |
608 """ | |
609 job_stats = Job(id=self.job.id).get_execution_details() | |
610 | |
611 subject = ('Autotest | Job ID: %s "%s" | Status: %s ' % | |
612 (self.job.id, self.job.name, status)) | |
613 | |
614 if hostname is not None: | |
615 subject += '| Hostname: %s ' % hostname | |
616 | |
617 if status not in ["1 Failed", "Failed"]: | |
618 subject += '| Success Rate: %.2f %%' % job_stats['success_rate'] | |
619 | |
620 body = "Job ID: %s\n" % self.job.id | |
621 body += "Job name: %s\n" % self.job.name | |
622 if hostname is not None: | |
623 body += "Host: %s\n" % hostname | |
624 if summary is not None: | |
625 body += "Summary: %s\n" % summary | |
626 body += "Status: %s\n" % status | |
627 body += "Results interface URL: %s\n" % self._view_job_url() | |
628 body += "Execution time (HH:MM:SS): %s\n" % job_stats['execution_time'] | |
629 if int(job_stats['total_executed']) > 0: | |
630 body += "User tests executed: %s\n" % job_stats['total_executed'] | |
631 body += "User tests passed: %s\n" % job_stats['total_passed'] | |
632 body += "User tests failed: %s\n" % job_stats['total_failed'] | |
633 body += ("User tests success rate: %.2f %%\n" % | |
634 job_stats['success_rate']) | |
635 | |
636 if job_stats['failed_rows']: | |
637 body += "Failures:\n" | |
638 body += job_stats['failed_rows'] | |
639 | |
640 return subject, body | |
641 | |
642 | |
643 def _email_on_status(self, status): | 594 def _email_on_status(self, status): |
644 hostname = self._get_hostname() | 595 hostname = self._get_hostname() |
645 subject, body = self._get_status_email_contents(status, None, hostname) | 596 |
| 597 subject = 'Autotest: Job ID: %s "%s" Host: %s %s' % ( |
| 598 self.job.id, self.job.name, hostname, status) |
| 599 body = "Job ID: %s\nJob Name: %s\nHost: %s\nStatus: %s\n%s\n" % ( |
| 600 self.job.id, self.job.name, hostname, status, |
| 601 self._view_job_url()) |
646 email_manager.manager.send_email(self.job.email_list, subject, body) | 602 email_manager.manager.send_email(self.job.email_list, subject, body) |
647 | 603 |
648 | 604 |
649 def _email_on_job_complete(self): | 605 def _email_on_job_complete(self): |
650 if not self.job.is_finished(): | 606 if not self.job.is_finished(): |
651 return | 607 return |
652 | 608 |
653 summary = [] | 609 summary_text = [] |
654 hosts_queue = HostQueueEntry.fetch('job_id = %s' % self.job.id) | 610 hosts_queue = HostQueueEntry.fetch('job_id = %s' % self.job.id) |
655 for queue_entry in hosts_queue: | 611 for queue_entry in hosts_queue: |
656 summary.append("Host: %s Status: %s" % | 612 summary_text.append("Host: %s Status: %s" % |
657 (queue_entry._get_hostname(), | 613 (queue_entry._get_hostname(), |
658 queue_entry.status)) | 614 queue_entry.status)) |
659 | 615 |
660 summary = "\n".join(summary) | 616 summary_text = "\n".join(summary_text) |
661 status_counts = models.Job.objects.get_status_counts( | 617 status_counts = models.Job.objects.get_status_counts( |
662 [self.job.id])[self.job.id] | 618 [self.job.id])[self.job.id] |
663 status = ', '.join('%d %s' % (count, status) for status, count | 619 status = ', '.join('%d %s' % (count, status) for status, count |
664 in status_counts.iteritems()) | 620 in status_counts.iteritems()) |
665 | 621 |
666 subject, body = self._get_status_email_contents(status, summary, None) | 622 subject = 'Autotest: Job ID: %s "%s" %s' % ( |
| 623 self.job.id, self.job.name, status) |
| 624 body = "Job ID: %s\nJob Name: %s\nStatus: %s\n%s\nSummary:\n%s" % ( |
| 625 self.job.id, self.job.name, status, self._view_job_url(), |
| 626 summary_text) |
667 email_manager.manager.send_email(self.job.email_list, subject, body) | 627 email_manager.manager.send_email(self.job.email_list, subject, body) |
668 | 628 |
669 | 629 |
670 def schedule_pre_job_tasks(self): | 630 def schedule_pre_job_tasks(self): |
671 logging.info("%s/%s/%s (job %s, entry %s) scheduled on %s, status=%s", | 631 logging.info("%s/%s/%s (job %s, entry %s) scheduled on %s, status=%s", |
672 self.job.name, self.meta_host, self.atomic_group_id, | 632 self.job.name, self.meta_host, self.atomic_group_id, |
673 self.job.id, self.id, self.host.hostname, self.status) | 633 self.job.id, self.id, self.host.hostname, self.status) |
674 | 634 |
675 self._do_schedule_pre_job_tasks() | 635 self._do_schedule_pre_job_tasks() |
676 | 636 |
(...skipping 177 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
854 SELECT * FROM afe_host_queue_entries | 814 SELECT * FROM afe_host_queue_entries |
855 WHERE job_id= %s | 815 WHERE job_id= %s |
856 """, (self.id,)) | 816 """, (self.id,)) |
857 entries = [HostQueueEntry(row=i) for i in rows] | 817 entries = [HostQueueEntry(row=i) for i in rows] |
858 | 818 |
859 assert len(entries)>0 | 819 assert len(entries)>0 |
860 | 820 |
861 return entries | 821 return entries |
862 | 822 |
863 | 823 |
864 def get_execution_details(self): | |
865 """ | |
866 Get test execution details for this job. | |
867 | |
868 @return: Dictionary with test execution details | |
869 """ | |
870 def _find_test_jobs(rows): | |
871 """ | |
872 Here we are looking for tests such as SERVER_JOB and CLIENT_JOB.* | |
873 Those are autotest 'internal job' tests, so they should not be | |
874 counted when evaluating the test stats. | |
875 | |
876 @param rows: List of rows (matrix) with database results. | |
877 """ | |
878 job_test_pattern = re.compile('SERVER|CLIENT\\_JOB\.[\d]') | |
879 n_test_jobs = 0 | |
880 for r in rows: | |
881 test_name = r[0] | |
882 if job_test_pattern.match(test_name): | |
883 n_test_jobs += 1 | |
884 | |
885 return n_test_jobs | |
886 | |
887 stats = {} | |
888 | |
889 rows = _db.execute(""" | |
890 SELECT t.test, s.word, t.reason | |
891 FROM tko_tests AS t, tko_jobs AS j, tko_status AS s | |
892 WHERE t.job_idx = j.job_idx | |
893 AND s.status_idx = t.status | |
894 AND j.afe_job_id = %s | |
895 """ % self.id) | |
896 | |
897 failed_rows = [r for r in rows if not 'GOOD' in r] | |
898 | |
899 n_test_jobs = _find_test_jobs(rows) | |
900 n_test_jobs_failed = _find_test_jobs(failed_rows) | |
901 | |
902 total_executed = len(rows) - n_test_jobs | |
903 total_failed = len(failed_rows) - n_test_jobs_failed | |
904 | |
905 if total_executed > 0: | |
906 success_rate = 100 - ((total_failed / float(total_executed)) * 100) | |
907 else: | |
908 success_rate = 0 | |
909 | |
910 stats['total_executed'] = total_executed | |
911 stats['total_failed'] = total_failed | |
912 stats['total_passed'] = total_executed - total_failed | |
913 stats['success_rate'] = success_rate | |
914 | |
915 status_header = ("Test Name", "Status", "Reason") | |
916 if failed_rows: | |
917 stats['failed_rows'] = utils.matrix_to_string(failed_rows, | |
918 status_header) | |
919 else: | |
920 stats['failed_rows'] = '' | |
921 | |
922 time_row = _db.execute(""" | |
923 SELECT started_time, finished_time | |
924 FROM tko_jobs | |
925 WHERE afe_job_id = %s | |
926 """ % self.id) | |
927 | |
928 if time_row: | |
929 t_begin, t_end = time_row[0] | |
930 delta = t_end - t_begin | |
931 minutes, seconds = divmod(delta.seconds, 60) | |
932 hours, minutes = divmod(minutes, 60) | |
933 stats['execution_time'] = ("%02d:%02d:%02d" % | |
934 (hours, minutes, seconds)) | |
935 else: | |
936 stats['execution_time'] = '(none)' | |
937 | |
938 return stats | |
939 | |
940 | |
941 def set_status(self, status, update_queues=False): | 824 def set_status(self, status, update_queues=False): |
942 self.update_field('status',status) | 825 self.update_field('status',status) |
943 | 826 |
944 if update_queues: | 827 if update_queues: |
945 for queue_entry in self.get_host_queue_entries(): | 828 for queue_entry in self.get_host_queue_entries(): |
946 queue_entry.set_status(status) | 829 queue_entry.set_status(status) |
947 | 830 |
948 | 831 |
949 def keyval_dict(self): | 832 def keyval_dict(self): |
950 return self.model().keyval_dict() | 833 return self.model().keyval_dict() |
(...skipping 353 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1304 def abort_delay_ready_task(self): | 1187 def abort_delay_ready_task(self): |
1305 """Abort the delayed task associated with this job, if any.""" | 1188 """Abort the delayed task associated with this job, if any.""" |
1306 if self._delay_ready_task: | 1189 if self._delay_ready_task: |
1307 # Cancel any pending callback that would try to run again | 1190 # Cancel any pending callback that would try to run again |
1308 # as we are already running. | 1191 # as we are already running. |
1309 self._delay_ready_task.abort() | 1192 self._delay_ready_task.abort() |
1310 | 1193 |
1311 | 1194 |
1312 def __str__(self): | 1195 def __str__(self): |
1313 return '%s-%s' % (self.id, self.owner) | 1196 return '%s-%s' % (self.id, self.owner) |
OLD | NEW |