OLD | NEW |
1 #!/usr/bin/python2.6 | 1 #!/usr/bin/python2.6 |
2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Program to run emerge in parallel, for significant speedup. | 6 """Program to run emerge in parallel, for significant speedup. |
7 | 7 |
8 Usage: | 8 Usage: |
9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] | 9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] |
10 [emerge args] package" | 10 [emerge args] package" |
(...skipping 1217 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1228 for i in sorted(deps_map): | 1228 for i in sorted(deps_map): |
1229 print "%s: (%s) needs" % (i, deps_map[i]["action"]) | 1229 print "%s: (%s) needs" % (i, deps_map[i]["action"]) |
1230 needs = deps_map[i]["needs"] | 1230 needs = deps_map[i]["needs"] |
1231 for j in sorted(needs): | 1231 for j in sorted(needs): |
1232 print " %s" % (j) | 1232 print " %s" % (j) |
1233 if not needs: | 1233 if not needs: |
1234 print " no dependencies" | 1234 print " no dependencies" |
1235 | 1235 |
1236 | 1236 |
1237 class EmergeJobState(object): | 1237 class EmergeJobState(object): |
1238 __slots__ = ["done", "filename", "last_output_seek", "last_output_timestamp", | 1238 __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek", |
1239 "pkgname", "retcode", "start_timestamp", "target"] | 1239 "last_output_timestamp", "pkgname", "retcode", "start_timestamp", |
| 1240 "target"] |
1240 | 1241 |
1241 def __init__(self, target, pkgname, done, filename, start_timestamp, | 1242 def __init__(self, target, pkgname, done, filename, start_timestamp, |
1242 retcode=None): | 1243 retcode=None): |
1243 | 1244 |
1244 # The full name of the target we're building (e.g. | 1245 # The full name of the target we're building (e.g. |
1245 # chromeos-base/chromeos-0.0.1-r60) | 1246 # chromeos-base/chromeos-0.0.1-r60) |
1246 self.target = target | 1247 self.target = target |
1247 | 1248 |
1248 # The short name of the target we're building (e.g. chromeos-0.0.1-r60) | 1249 # The short name of the target we're building (e.g. chromeos-0.0.1-r60) |
1249 self.pkgname = pkgname | 1250 self.pkgname = pkgname |
1250 | 1251 |
1251 # Whether the job is done. (True if the job is done; false otherwise.) | 1252 # Whether the job is done. (True if the job is done; false otherwise.) |
1252 self.done = done | 1253 self.done = done |
1253 | 1254 |
1254 # The filename where output is currently stored. | 1255 # The filename where output is currently stored. |
1255 self.filename = filename | 1256 self.filename = filename |
1256 | 1257 |
| 1258 # The timestamp of the last time we printed the name of the log file. We |
| 1259 # print this at the beginning of the job, so this starts at |
| 1260 # start_timestamp. |
| 1261 self.last_notify_timestamp = start_timestamp |
| 1262 |
1257 # The location (in bytes) of the end of the last complete line we printed. | 1263 # The location (in bytes) of the end of the last complete line we printed. |
1258 # This starts off at zero. We use this to jump to the right place when we | 1264 # This starts off at zero. We use this to jump to the right place when we |
1259 # print output from the same ebuild multiple times. | 1265 # print output from the same ebuild multiple times. |
1260 self.last_output_seek = 0 | 1266 self.last_output_seek = 0 |
1261 | 1267 |
1262 # The timestamp of the last time we printed output. Since we haven't | 1268 # The timestamp of the last time we printed output. Since we haven't |
1263 # printed output yet, this starts at zero. | 1269 # printed output yet, this starts at zero. |
1264 self.last_output_timestamp = 0 | 1270 self.last_output_timestamp = 0 |
1265 | 1271 |
1266 # The return code of our job, if the job is actually finished. | 1272 # The return code of our job, if the job is actually finished. |
(...skipping 275 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1542 loads = open("/proc/loadavg", "r").readline().split()[:3] | 1548 loads = open("/proc/loadavg", "r").readline().split()[:3] |
1543 return " ".join(loads) | 1549 return " ".join(loads) |
1544 | 1550 |
1545 def _Print(self, line): | 1551 def _Print(self, line): |
1546 """Print a single line.""" | 1552 """Print a single line.""" |
1547 self._print_queue.put(LinePrinter(line)) | 1553 self._print_queue.put(LinePrinter(line)) |
1548 | 1554 |
1549 def _Status(self): | 1555 def _Status(self): |
1550 """Print status.""" | 1556 """Print status.""" |
1551 current_time = time.time() | 1557 current_time = time.time() |
1552 seconds = current_time - GLOBAL_START | 1558 no_output = True |
1553 line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s " | |
1554 "[Time %dm%.1fs Load %s]") | |
1555 qsize = self._emerge_queue.qsize() | |
1556 self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize, | |
1557 len(self._retry_queue), self._total_jobs, | |
1558 seconds / 60, seconds % 60, self._LoadAvg())) | |
1559 | 1559 |
1560 # Print interim output every minute if --show-output is used. Otherwise, | 1560 # Print interim output every minute if --show-output is used. Otherwise, |
1561 # only print output if a job has been running for 60 minutes or more. | 1561 # print notifications about running packages every 2 minutes, and print |
| 1562 # full output for jobs that have been running for 60 minutes or more. |
1562 if self._show_output: | 1563 if self._show_output: |
1563 interval = 60 | 1564 interval = 60 |
| 1565 notify_interval = 0 |
1564 else: | 1566 else: |
1565 interval = 60 * 60 | 1567 interval = 60 * 60 |
| 1568 notify_interval = 60 * 2 |
1566 for target, job in self._jobs.iteritems(): | 1569 for target, job in self._jobs.iteritems(): |
1567 if job: | 1570 if job: |
1568 last_timestamp = max(job.start_timestamp, job.last_output_timestamp) | 1571 last_timestamp = max(job.start_timestamp, job.last_output_timestamp) |
1569 if last_timestamp + interval < current_time: | 1572 if last_timestamp + interval < current_time: |
1570 self._print_queue.put(JobPrinter(job)) | 1573 self._print_queue.put(JobPrinter(job)) |
1571 job.last_output_timestamp = current_time | 1574 job.last_output_timestamp = current_time |
| 1575 no_output = False |
| 1576 elif (notify_interval and |
| 1577 job.last_notify_timestamp + notify_interval < current_time): |
| 1578 job_seconds = current_time - job.start_timestamp |
| 1579 args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename) |
| 1580 info = "Still building %s (%dm%.1fs). Logs in %s" % args |
| 1581 job.last_notify_timestamp = current_time |
| 1582 self._Print(info) |
| 1583 no_output = False |
| 1584 |
| 1585 # If we haven't printed any messages yet, print a general status message |
| 1586 # here. |
| 1587 if no_output: |
| 1588 seconds = current_time - GLOBAL_START |
| 1589 line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s " |
| 1590 "[Time %dm%.1fs Load %s]") |
| 1591 qsize = self._emerge_queue.qsize() |
| 1592 self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize, |
| 1593 len(self._retry_queue), self._total_jobs, |
| 1594 seconds / 60, seconds % 60, self._LoadAvg())) |
1572 | 1595 |
1573 def _Finish(self, target): | 1596 def _Finish(self, target): |
1574 """Mark a target as completed and unblock dependecies.""" | 1597 """Mark a target as completed and unblock dependecies.""" |
1575 for dep in self._deps_map[target]["provides"]: | 1598 for dep in self._deps_map[target]["provides"]: |
1576 del self._deps_map[dep]["needs"][target] | 1599 del self._deps_map[dep]["needs"][target] |
1577 if not self._deps_map[dep]["needs"]: | 1600 if not self._deps_map[dep]["needs"]: |
1578 self._Schedule(dep) | 1601 self._Schedule(dep) |
1579 self._deps_map.pop(target) | 1602 self._deps_map.pop(target) |
1580 | 1603 |
1581 def _Retry(self): | 1604 def _Retry(self): |
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1766 # need to upgrade the rest of the packages. So we'll go ahead and do that. | 1789 # need to upgrade the rest of the packages. So we'll go ahead and do that. |
1767 if portage_upgrade: | 1790 if portage_upgrade: |
1768 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"] | 1791 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"] |
1769 os.execvp(os.path.realpath(sys.argv[0]), args) | 1792 os.execvp(os.path.realpath(sys.argv[0]), args) |
1770 | 1793 |
1771 print "Done" | 1794 print "Done" |
1772 sys.exit(0) | 1795 sys.exit(0) |
1773 | 1796 |
1774 if __name__ == "__main__": | 1797 if __name__ == "__main__": |
1775 main() | 1798 main() |
OLD | NEW |