| OLD | NEW |
| 1 #!/usr/bin/python2.6 | 1 #!/usr/bin/python2.6 |
| 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Program to run emerge in parallel, for significant speedup. | 6 """Program to run emerge in parallel, for significant speedup. |
| 7 | 7 |
| 8 Usage: | 8 Usage: |
| 9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] | 9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] |
| 10 [emerge args] package" | 10 [emerge args] package" |
| (...skipping 1217 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1228 for i in sorted(deps_map): | 1228 for i in sorted(deps_map): |
| 1229 print "%s: (%s) needs" % (i, deps_map[i]["action"]) | 1229 print "%s: (%s) needs" % (i, deps_map[i]["action"]) |
| 1230 needs = deps_map[i]["needs"] | 1230 needs = deps_map[i]["needs"] |
| 1231 for j in sorted(needs): | 1231 for j in sorted(needs): |
| 1232 print " %s" % (j) | 1232 print " %s" % (j) |
| 1233 if not needs: | 1233 if not needs: |
| 1234 print " no dependencies" | 1234 print " no dependencies" |
| 1235 | 1235 |
| 1236 | 1236 |
| 1237 class EmergeJobState(object): | 1237 class EmergeJobState(object): |
| 1238 __slots__ = ["done", "filename", "last_output_seek", "last_output_timestamp", | 1238 __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek", |
| 1239 "pkgname", "retcode", "start_timestamp", "target"] | 1239 "last_output_timestamp", "pkgname", "retcode", "start_timestamp", |
| 1240 "target"] |
| 1240 | 1241 |
| 1241 def __init__(self, target, pkgname, done, filename, start_timestamp, | 1242 def __init__(self, target, pkgname, done, filename, start_timestamp, |
| 1242 retcode=None): | 1243 retcode=None): |
| 1243 | 1244 |
| 1244 # The full name of the target we're building (e.g. | 1245 # The full name of the target we're building (e.g. |
| 1245 # chromeos-base/chromeos-0.0.1-r60) | 1246 # chromeos-base/chromeos-0.0.1-r60) |
| 1246 self.target = target | 1247 self.target = target |
| 1247 | 1248 |
| 1248 # The short name of the target we're building (e.g. chromeos-0.0.1-r60) | 1249 # The short name of the target we're building (e.g. chromeos-0.0.1-r60) |
| 1249 self.pkgname = pkgname | 1250 self.pkgname = pkgname |
| 1250 | 1251 |
| 1251 # Whether the job is done. (True if the job is done; false otherwise.) | 1252 # Whether the job is done. (True if the job is done; false otherwise.) |
| 1252 self.done = done | 1253 self.done = done |
| 1253 | 1254 |
| 1254 # The filename where output is currently stored. | 1255 # The filename where output is currently stored. |
| 1255 self.filename = filename | 1256 self.filename = filename |
| 1256 | 1257 |
| 1258 # The timestamp of the last time we printed the name of the log file. We |
| 1259 # print this at the beginning of the job, so this starts at |
| 1260 # start_timestamp. |
| 1261 self.last_notify_timestamp = start_timestamp |
| 1262 |
| 1257 # The location (in bytes) of the end of the last complete line we printed. | 1263 # The location (in bytes) of the end of the last complete line we printed. |
| 1258 # This starts off at zero. We use this to jump to the right place when we | 1264 # This starts off at zero. We use this to jump to the right place when we |
| 1259 # print output from the same ebuild multiple times. | 1265 # print output from the same ebuild multiple times. |
| 1260 self.last_output_seek = 0 | 1266 self.last_output_seek = 0 |
| 1261 | 1267 |
| 1262 # The timestamp of the last time we printed output. Since we haven't | 1268 # The timestamp of the last time we printed output. Since we haven't |
| 1263 # printed output yet, this starts at zero. | 1269 # printed output yet, this starts at zero. |
| 1264 self.last_output_timestamp = 0 | 1270 self.last_output_timestamp = 0 |
| 1265 | 1271 |
| 1266 # The return code of our job, if the job is actually finished. | 1272 # The return code of our job, if the job is actually finished. |
| (...skipping 275 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1542 loads = open("/proc/loadavg", "r").readline().split()[:3] | 1548 loads = open("/proc/loadavg", "r").readline().split()[:3] |
| 1543 return " ".join(loads) | 1549 return " ".join(loads) |
| 1544 | 1550 |
| 1545 def _Print(self, line): | 1551 def _Print(self, line): |
| 1546 """Print a single line.""" | 1552 """Print a single line.""" |
| 1547 self._print_queue.put(LinePrinter(line)) | 1553 self._print_queue.put(LinePrinter(line)) |
| 1548 | 1554 |
| 1549 def _Status(self): | 1555 def _Status(self): |
| 1550 """Print status.""" | 1556 """Print status.""" |
| 1551 current_time = time.time() | 1557 current_time = time.time() |
| 1552 seconds = current_time - GLOBAL_START | 1558 no_output = True |
| 1553 line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s " | |
| 1554 "[Time %dm%.1fs Load %s]") | |
| 1555 qsize = self._emerge_queue.qsize() | |
| 1556 self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize, | |
| 1557 len(self._retry_queue), self._total_jobs, | |
| 1558 seconds / 60, seconds % 60, self._LoadAvg())) | |
| 1559 | 1559 |
| 1560 # Print interim output every minute if --show-output is used. Otherwise, | 1560 # Print interim output every minute if --show-output is used. Otherwise, |
| 1561 # only print output if a job has been running for 60 minutes or more. | 1561 # print notifications about running packages every 2 minutes, and print |
| 1562 # full output for jobs that have been running for 60 minutes or more. |
| 1562 if self._show_output: | 1563 if self._show_output: |
| 1563 interval = 60 | 1564 interval = 60 |
| 1565 notify_interval = 0 |
| 1564 else: | 1566 else: |
| 1565 interval = 60 * 60 | 1567 interval = 60 * 60 |
| 1568 notify_interval = 60 * 2 |
| 1566 for target, job in self._jobs.iteritems(): | 1569 for target, job in self._jobs.iteritems(): |
| 1567 if job: | 1570 if job: |
| 1568 last_timestamp = max(job.start_timestamp, job.last_output_timestamp) | 1571 last_timestamp = max(job.start_timestamp, job.last_output_timestamp) |
| 1569 if last_timestamp + interval < current_time: | 1572 if last_timestamp + interval < current_time: |
| 1570 self._print_queue.put(JobPrinter(job)) | 1573 self._print_queue.put(JobPrinter(job)) |
| 1571 job.last_output_timestamp = current_time | 1574 job.last_output_timestamp = current_time |
| 1575 no_output = False |
| 1576 elif (notify_interval and |
| 1577 job.last_notify_timestamp + notify_interval < current_time): |
| 1578 job_seconds = current_time - job.start_timestamp |
| 1579 args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename) |
| 1580 info = "Still building %s (%dm%.1fs). Logs in %s" % args |
| 1581 job.last_notify_timestamp = current_time |
| 1582 self._Print(info) |
| 1583 no_output = False |
| 1584 |
| 1585 # If we haven't printed any messages yet, print a general status message |
| 1586 # here. |
| 1587 if no_output: |
| 1588 seconds = current_time - GLOBAL_START |
| 1589 line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s " |
| 1590 "[Time %dm%.1fs Load %s]") |
| 1591 qsize = self._emerge_queue.qsize() |
| 1592 self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize, |
| 1593 len(self._retry_queue), self._total_jobs, |
| 1594 seconds / 60, seconds % 60, self._LoadAvg())) |
| 1572 | 1595 |
| 1573 def _Finish(self, target): | 1596 def _Finish(self, target): |
| 1574 """Mark a target as completed and unblock dependecies.""" | 1597 """Mark a target as completed and unblock dependecies.""" |
| 1575 for dep in self._deps_map[target]["provides"]: | 1598 for dep in self._deps_map[target]["provides"]: |
| 1576 del self._deps_map[dep]["needs"][target] | 1599 del self._deps_map[dep]["needs"][target] |
| 1577 if not self._deps_map[dep]["needs"]: | 1600 if not self._deps_map[dep]["needs"]: |
| 1578 self._Schedule(dep) | 1601 self._Schedule(dep) |
| 1579 self._deps_map.pop(target) | 1602 self._deps_map.pop(target) |
| 1580 | 1603 |
| 1581 def _Retry(self): | 1604 def _Retry(self): |
| (...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1766 # need to upgrade the rest of the packages. So we'll go ahead and do that. | 1789 # need to upgrade the rest of the packages. So we'll go ahead and do that. |
| 1767 if portage_upgrade: | 1790 if portage_upgrade: |
| 1768 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"] | 1791 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"] |
| 1769 os.execvp(os.path.realpath(sys.argv[0]), args) | 1792 os.execvp(os.path.realpath(sys.argv[0]), args) |
| 1770 | 1793 |
| 1771 print "Done" | 1794 print "Done" |
| 1772 sys.exit(0) | 1795 sys.exit(0) |
| 1773 | 1796 |
| 1774 if __name__ == "__main__": | 1797 if __name__ == "__main__": |
| 1775 main() | 1798 main() |
| OLD | NEW |