Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(156)

Side by Side Diff: parallel_emerge

Issue 4318003: Print status messages in parallel_emerge about jobs that are longer than 2 mins. (Closed) Base URL: ssh://git@gitrw.chromium.org:9222/crosutils.git
Patch Set: s/full outputs/full output/ Created 10 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python2.6 1 #!/usr/bin/python2.6
2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Program to run emerge in parallel, for significant speedup. 6 """Program to run emerge in parallel, for significant speedup.
7 7
8 Usage: 8 Usage:
9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] 9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps]
10 [emerge args] package" 10 [emerge args] package"
(...skipping 1217 matching lines...) Expand 10 before | Expand all | Expand 10 after
1228 for i in sorted(deps_map): 1228 for i in sorted(deps_map):
1229 print "%s: (%s) needs" % (i, deps_map[i]["action"]) 1229 print "%s: (%s) needs" % (i, deps_map[i]["action"])
1230 needs = deps_map[i]["needs"] 1230 needs = deps_map[i]["needs"]
1231 for j in sorted(needs): 1231 for j in sorted(needs):
1232 print " %s" % (j) 1232 print " %s" % (j)
1233 if not needs: 1233 if not needs:
1234 print " no dependencies" 1234 print " no dependencies"
1235 1235
1236 1236
1237 class EmergeJobState(object): 1237 class EmergeJobState(object):
1238 __slots__ = ["done", "filename", "last_output_seek", "last_output_timestamp", 1238 __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek",
1239 "pkgname", "retcode", "start_timestamp", "target"] 1239 "last_output_timestamp", "pkgname", "retcode", "start_timestamp",
1240 "target"]
1240 1241
1241 def __init__(self, target, pkgname, done, filename, start_timestamp, 1242 def __init__(self, target, pkgname, done, filename, start_timestamp,
1242 retcode=None): 1243 retcode=None):
1243 1244
1244 # The full name of the target we're building (e.g. 1245 # The full name of the target we're building (e.g.
1245 # chromeos-base/chromeos-0.0.1-r60) 1246 # chromeos-base/chromeos-0.0.1-r60)
1246 self.target = target 1247 self.target = target
1247 1248
1248 # The short name of the target we're building (e.g. chromeos-0.0.1-r60) 1249 # The short name of the target we're building (e.g. chromeos-0.0.1-r60)
1249 self.pkgname = pkgname 1250 self.pkgname = pkgname
1250 1251
1251 # Whether the job is done. (True if the job is done; false otherwise.) 1252 # Whether the job is done. (True if the job is done; false otherwise.)
1252 self.done = done 1253 self.done = done
1253 1254
1254 # The filename where output is currently stored. 1255 # The filename where output is currently stored.
1255 self.filename = filename 1256 self.filename = filename
1256 1257
1258 # The timestamp of the last time we printed the name of the log file. We
1259 # print this at the beginning of the job, so this starts at
1260 # start_timestamp.
1261 self.last_notify_timestamp = start_timestamp
1262
1257 # The location (in bytes) of the end of the last complete line we printed. 1263 # The location (in bytes) of the end of the last complete line we printed.
1258 # This starts off at zero. We use this to jump to the right place when we 1264 # This starts off at zero. We use this to jump to the right place when we
1259 # print output from the same ebuild multiple times. 1265 # print output from the same ebuild multiple times.
1260 self.last_output_seek = 0 1266 self.last_output_seek = 0
1261 1267
1262 # The timestamp of the last time we printed output. Since we haven't 1268 # The timestamp of the last time we printed output. Since we haven't
1263 # printed output yet, this starts at zero. 1269 # printed output yet, this starts at zero.
1264 self.last_output_timestamp = 0 1270 self.last_output_timestamp = 0
1265 1271
1266 # The return code of our job, if the job is actually finished. 1272 # The return code of our job, if the job is actually finished.
(...skipping 275 matching lines...) Expand 10 before | Expand all | Expand 10 after
1542 loads = open("/proc/loadavg", "r").readline().split()[:3] 1548 loads = open("/proc/loadavg", "r").readline().split()[:3]
1543 return " ".join(loads) 1549 return " ".join(loads)
1544 1550
1545 def _Print(self, line): 1551 def _Print(self, line):
1546 """Print a single line.""" 1552 """Print a single line."""
1547 self._print_queue.put(LinePrinter(line)) 1553 self._print_queue.put(LinePrinter(line))
1548 1554
1549 def _Status(self): 1555 def _Status(self):
1550 """Print status.""" 1556 """Print status."""
1551 current_time = time.time() 1557 current_time = time.time()
1552 seconds = current_time - GLOBAL_START 1558 no_output = True
1553 line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s "
1554 "[Time %dm%.1fs Load %s]")
1555 qsize = self._emerge_queue.qsize()
1556 self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize,
1557 len(self._retry_queue), self._total_jobs,
1558 seconds / 60, seconds % 60, self._LoadAvg()))
1559 1559
1560 # Print interim output every minute if --show-output is used. Otherwise, 1560 # Print interim output every minute if --show-output is used. Otherwise,
1561 # only print output if a job has been running for 60 minutes or more. 1561 # print notifications about running packages every 2 minutes, and print
1562 # full output for jobs that have been running for 60 minutes or more.
1562 if self._show_output: 1563 if self._show_output:
1563 interval = 60 1564 interval = 60
1565 notify_interval = 0
1564 else: 1566 else:
1565 interval = 60 * 60 1567 interval = 60 * 60
1568 notify_interval = 60 * 2
1566 for target, job in self._jobs.iteritems(): 1569 for target, job in self._jobs.iteritems():
1567 if job: 1570 if job:
1568 last_timestamp = max(job.start_timestamp, job.last_output_timestamp) 1571 last_timestamp = max(job.start_timestamp, job.last_output_timestamp)
1569 if last_timestamp + interval < current_time: 1572 if last_timestamp + interval < current_time:
1570 self._print_queue.put(JobPrinter(job)) 1573 self._print_queue.put(JobPrinter(job))
1571 job.last_output_timestamp = current_time 1574 job.last_output_timestamp = current_time
1575 no_output = False
1576 elif (notify_interval and
1577 job.last_notify_timestamp + notify_interval < current_time):
1578 job_seconds = current_time - job.start_timestamp
1579 args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename)
1580 info = "Still building %s (%dm%.1fs). Logs in %s" % args
1581 job.last_notify_timestamp = current_time
1582 self._Print(info)
1583 no_output = False
1584
1585 # If we haven't printed any messages yet, print a general status message
1586 # here.
1587 if no_output:
1588 seconds = current_time - GLOBAL_START
1589 line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s "
1590 "[Time %dm%.1fs Load %s]")
1591 qsize = self._emerge_queue.qsize()
1592 self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize,
1593 len(self._retry_queue), self._total_jobs,
1594 seconds / 60, seconds % 60, self._LoadAvg()))
1572 1595
1573 def _Finish(self, target): 1596 def _Finish(self, target):
1574 """Mark a target as completed and unblock dependecies.""" 1597 """Mark a target as completed and unblock dependecies."""
1575 for dep in self._deps_map[target]["provides"]: 1598 for dep in self._deps_map[target]["provides"]:
1576 del self._deps_map[dep]["needs"][target] 1599 del self._deps_map[dep]["needs"][target]
1577 if not self._deps_map[dep]["needs"]: 1600 if not self._deps_map[dep]["needs"]:
1578 self._Schedule(dep) 1601 self._Schedule(dep)
1579 self._deps_map.pop(target) 1602 self._deps_map.pop(target)
1580 1603
1581 def _Retry(self): 1604 def _Retry(self):
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after
1766 # need to upgrade the rest of the packages. So we'll go ahead and do that. 1789 # need to upgrade the rest of the packages. So we'll go ahead and do that.
1767 if portage_upgrade: 1790 if portage_upgrade:
1768 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"] 1791 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"]
1769 os.execvp(os.path.realpath(sys.argv[0]), args) 1792 os.execvp(os.path.realpath(sys.argv[0]), args)
1770 1793
1771 print "Done" 1794 print "Done"
1772 sys.exit(0) 1795 sys.exit(0)
1773 1796
1774 if __name__ == "__main__": 1797 if __name__ == "__main__":
1775 main() 1798 main()
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698