OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright 2014 the V8 project authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 # This script retrieves the history of all V8 branches and | |
7 # their corresponding Chromium revisions. | |
8 | |
9 # Requires a chromium checkout with branch heads: | |
10 # gclient sync --with_branch_heads | |
11 # gclient fetch | |
12 | |
13 import argparse | |
14 import csv | |
15 import itertools | |
16 import json | |
17 import os | |
18 import re | |
19 import sys | |
20 | |
21 from common_includes import * | |
22 | |
23 CONFIG = { | |
24 "BRANCHNAME": "retrieve-v8-releases", | |
25 "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile", | |
26 } | |
27 | |
28 # Expression for retrieving the bleeding edge revision from a commit message. | |
29 PUSH_MSG_SVN_RE = re.compile(r".* \(based on bleeding_edge revision r(\d+)\)$") | |
30 PUSH_MSG_GIT_RE = re.compile(r".* \(based on ([a-fA-F0-9]+)\)$") | |
31 | |
32 # Expression for retrieving the merged patches from a merge commit message | |
33 # (old and new format). | |
34 MERGE_MESSAGE_RE = re.compile(r"^.*[M|m]erged (.+)(\)| into).*$", re.M) | |
35 | |
36 CHERRY_PICK_TITLE_GIT_RE = re.compile(r"^.* \(cherry\-pick\)\.?$") | |
37 | |
38 # New git message for cherry-picked CLs. One message per line. | |
39 MERGE_MESSAGE_GIT_RE = re.compile(r"^Merged ([a-fA-F0-9]+)\.?$") | |
40 | |
41 # Expression for retrieving reverted patches from a commit message (old and | |
42 # new format). | |
43 ROLLBACK_MESSAGE_RE = re.compile(r"^.*[R|r]ollback of (.+)(\)| in).*$", re.M) | |
44 | |
45 # New git message for reverted CLs. One message per line. | |
46 ROLLBACK_MESSAGE_GIT_RE = re.compile(r"^Rollback of ([a-fA-F0-9]+)\.?$") | |
47 | |
48 # Expression for retrieving the code review link. | |
49 REVIEW_LINK_RE = re.compile(r"^Review URL: (.+)$", re.M) | |
50 | |
51 # Expression with three versions (historical) for extracting the v8 revision | |
52 # from the chromium DEPS file. | |
53 DEPS_RE = re.compile(r"""^\s*(?:["']v8_revision["']: ["']""" | |
54 """|\(Var\("googlecode_url"\) % "v8"\) \+ "\/trunk@""" | |
55 """|"http\:\/\/v8\.googlecode\.com\/svn\/trunk@)""" | |
56 """([^"']+)["'].*$""", re.M) | |
57 | |
58 # Expression to pick tag and revision for bleeding edge tags. To be used with | |
59 # output of 'svn log'. | |
60 BLEEDING_EDGE_TAGS_RE = re.compile( | |
61 r"A \/tags\/([^\s]+) \(from \/branches\/bleeding_edge\:(\d+)\)") | |
62 | |
63 | |
64 def SortBranches(branches): | |
65 """Sort branches with version number names.""" | |
66 return sorted(branches, key=SortingKey, reverse=True) | |
67 | |
68 | |
69 def FilterDuplicatesAndReverse(cr_releases): | |
70 """Returns the chromium releases in reverse order filtered by v8 revision | |
71 duplicates. | |
72 | |
73 cr_releases is a list of [cr_rev, v8_rev] reverse-sorted by cr_rev. | |
74 """ | |
75 last = "" | |
76 result = [] | |
77 for release in reversed(cr_releases): | |
78 if last == release[1]: | |
79 continue | |
80 last = release[1] | |
81 result.append(release) | |
82 return result | |
83 | |
84 | |
85 def BuildRevisionRanges(cr_releases): | |
86 """Returns a mapping of v8 revision -> chromium ranges. | |
87 The ranges are comma-separated, each range has the form R1:R2. The newest | |
88 entry is the only one of the form R1, as there is no end range. | |
89 | |
90 cr_releases is a list of [cr_rev, v8_rev] reverse-sorted by cr_rev. | |
91 cr_rev either refers to a chromium svn revision or a chromium branch number. | |
92 """ | |
93 range_lists = {} | |
94 cr_releases = FilterDuplicatesAndReverse(cr_releases) | |
95 | |
96 # Visit pairs of cr releases from oldest to newest. | |
97 for cr_from, cr_to in itertools.izip( | |
98 cr_releases, itertools.islice(cr_releases, 1, None)): | |
99 | |
100 # Assume the chromium revisions are all different. | |
101 assert cr_from[0] != cr_to[0] | |
102 | |
103 # TODO(machenbach): Subtraction is not git friendly. | |
104 ran = "%s:%d" % (cr_from[0], int(cr_to[0]) - 1) | |
105 | |
106 # Collect the ranges in lists per revision. | |
107 range_lists.setdefault(cr_from[1], []).append(ran) | |
108 | |
109 # Add the newest revision. | |
110 if cr_releases: | |
111 range_lists.setdefault(cr_releases[-1][1], []).append(cr_releases[-1][0]) | |
112 | |
113 # Stringify and comma-separate the range lists. | |
114 return dict((rev, ", ".join(ran)) for rev, ran in range_lists.iteritems()) | |
115 | |
116 | |
117 def MatchSafe(match): | |
118 if match: | |
119 return match.group(1) | |
120 else: | |
121 return "" | |
122 | |
123 | |
124 class Preparation(Step): | |
125 MESSAGE = "Preparation." | |
126 | |
127 def RunStep(self): | |
128 self.CommonPrepare() | |
129 self.PrepareBranch() | |
130 | |
131 | |
132 class RetrieveV8Releases(Step): | |
133 MESSAGE = "Retrieve all V8 releases." | |
134 | |
135 def ExceedsMax(self, releases): | |
136 return (self._options.max_releases > 0 | |
137 and len(releases) > self._options.max_releases) | |
138 | |
139 def GetMasterHashFromPush(self, title): | |
140 return MatchSafe(PUSH_MSG_GIT_RE.match(title)) | |
141 | |
142 def GetMergedPatches(self, body): | |
143 patches = MatchSafe(MERGE_MESSAGE_RE.search(body)) | |
144 if not patches: | |
145 patches = MatchSafe(ROLLBACK_MESSAGE_RE.search(body)) | |
146 if patches: | |
147 # Indicate reverted patches with a "-". | |
148 patches = "-%s" % patches | |
149 return patches | |
150 | |
151 def GetMergedPatchesGit(self, body): | |
152 patches = [] | |
153 for line in body.splitlines(): | |
154 patch = MatchSafe(MERGE_MESSAGE_GIT_RE.match(line)) | |
155 if patch: | |
156 patches.append(patch) | |
157 patch = MatchSafe(ROLLBACK_MESSAGE_GIT_RE.match(line)) | |
158 if patch: | |
159 patches.append("-%s" % patch) | |
160 return ", ".join(patches) | |
161 | |
162 | |
163 def GetReleaseDict( | |
164 self, git_hash, master_position, master_hash, branch, version, | |
165 patches, cl_body): | |
166 revision = self.GetCommitPositionNumber(git_hash) | |
167 return { | |
168 # The cr commit position number on the branch. | |
169 "revision": revision, | |
170 # The git revision on the branch. | |
171 "revision_git": git_hash, | |
172 # The cr commit position number on master. | |
173 "bleeding_edge": master_position, | |
174 # The same for git. | |
175 "bleeding_edge_git": master_hash, | |
176 # The branch name. | |
177 "branch": branch, | |
178 # The version for displaying in the form 3.26.3 or 3.26.3.12. | |
179 "version": version, | |
180 # The date of the commit. | |
181 "date": self.GitLog(n=1, format="%ci", git_hash=git_hash), | |
182 # Merged patches if available in the form 'r1234, r2345'. | |
183 "patches_merged": patches, | |
184 # Default for easier output formatting. | |
185 "chromium_revision": "", | |
186 # Default for easier output formatting. | |
187 "chromium_branch": "", | |
188 # Link to the CL on code review. Candiates pushes are not uploaded, | |
189 # so this field will be populated below with the recent roll CL link. | |
190 "review_link": MatchSafe(REVIEW_LINK_RE.search(cl_body)), | |
191 # Link to the commit message on google code. | |
192 "revision_link": ("https://code.google.com/p/v8/source/detail?r=%s" | |
193 % revision), | |
194 } | |
195 | |
196 def GetRelease(self, git_hash, branch): | |
197 self.ReadAndPersistVersion() | |
198 base_version = [self["major"], self["minor"], self["build"]] | |
199 version = ".".join(base_version) | |
200 body = self.GitLog(n=1, format="%B", git_hash=git_hash) | |
201 | |
202 patches = "" | |
203 if self["patch"] != "0": | |
204 version += ".%s" % self["patch"] | |
205 if CHERRY_PICK_TITLE_GIT_RE.match(body.splitlines()[0]): | |
206 patches = self.GetMergedPatchesGit(body) | |
207 else: | |
208 patches = self.GetMergedPatches(body) | |
209 | |
210 title = self.GitLog(n=1, format="%s", git_hash=git_hash) | |
211 master_hash = self.GetMasterHashFromPush(title) | |
212 master_position = "" | |
213 if master_hash: | |
214 master_position = self.GetCommitPositionNumber(master_hash) | |
215 # TODO(machenbach): Add the commit position number. | |
216 return self.GetReleaseDict( | |
217 git_hash, master_position, master_hash, branch, version, | |
218 patches, body), self["patch"] | |
219 | |
220 def GetReleasesFromMaster(self): | |
221 # TODO(machenbach): Implement this in git as soon as we tag again on | |
222 # master. | |
223 # tag_text = self.SVN("log https://v8.googlecode.com/svn/tags -v | |
224 # --limit 20") | |
225 # releases = [] | |
226 # for (tag, revision) in re.findall(BLEEDING_EDGE_TAGS_RE, tag_text): | |
227 # git_hash = self.vc.SvnGit(revision) | |
228 | |
229 # Add bleeding edge release. It does not contain patches or a code | |
230 # review link, as tags are not uploaded. | |
231 # releases.append(self.GetReleaseDict( | |
232 # git_hash, revision, git_hash, self.vc.MasterBranch(), tag, "", "")) | |
233 return [] | |
234 | |
235 def GetReleasesFromBranch(self, branch): | |
236 self.GitReset(self.vc.RemoteBranch(branch)) | |
237 if branch == self.vc.MasterBranch(): | |
238 return self.GetReleasesFromMaster() | |
239 | |
240 releases = [] | |
241 try: | |
242 for git_hash in self.GitLog(format="%H").splitlines(): | |
243 if VERSION_FILE not in self.GitChangedFiles(git_hash): | |
244 continue | |
245 if self.ExceedsMax(releases): | |
246 break # pragma: no cover | |
247 if not self.GitCheckoutFileSafe(VERSION_FILE, git_hash): | |
248 break # pragma: no cover | |
249 | |
250 release, patch_level = self.GetRelease(git_hash, branch) | |
251 releases.append(release) | |
252 | |
253 # Follow branches only until their creation point. | |
254 # TODO(machenbach): This omits patches if the version file wasn't | |
255 # manipulated correctly. Find a better way to detect the point where | |
256 # the parent of the branch head leads to the trunk branch. | |
257 if branch != self.vc.CandidateBranch() and patch_level == "0": | |
258 break | |
259 | |
260 # Allow Ctrl-C interrupt. | |
261 except (KeyboardInterrupt, SystemExit): # pragma: no cover | |
262 pass | |
263 | |
264 # Clean up checked-out version file. | |
265 self.GitCheckoutFileSafe(VERSION_FILE, "HEAD") | |
266 return releases | |
267 | |
268 def RunStep(self): | |
269 self.GitCreateBranch(self._config["BRANCHNAME"]) | |
270 branches = self.vc.GetBranches() | |
271 releases = [] | |
272 if self._options.branch == 'recent': | |
273 # Get only recent development on candidates, beta and stable. | |
274 if self._options.max_releases == 0: # pragma: no cover | |
275 self._options.max_releases = 10 | |
276 beta, stable = SortBranches(branches)[0:2] | |
277 releases += self.GetReleasesFromBranch(stable) | |
278 releases += self.GetReleasesFromBranch(beta) | |
279 releases += self.GetReleasesFromBranch(self.vc.CandidateBranch()) | |
280 releases += self.GetReleasesFromBranch(self.vc.MasterBranch()) | |
281 elif self._options.branch == 'all': # pragma: no cover | |
282 # Retrieve the full release history. | |
283 for branch in branches: | |
284 releases += self.GetReleasesFromBranch(branch) | |
285 releases += self.GetReleasesFromBranch(self.vc.CandidateBranch()) | |
286 releases += self.GetReleasesFromBranch(self.vc.MasterBranch()) | |
287 else: # pragma: no cover | |
288 # Retrieve history for a specified branch. | |
289 assert self._options.branch in (branches + | |
290 [self.vc.CandidateBranch(), self.vc.MasterBranch()]) | |
291 releases += self.GetReleasesFromBranch(self._options.branch) | |
292 | |
293 self["releases"] = sorted(releases, | |
294 key=lambda r: SortingKey(r["version"]), | |
295 reverse=True) | |
296 | |
297 | |
298 class SwitchChromium(Step): | |
299 MESSAGE = "Switch to Chromium checkout." | |
300 | |
301 def RunStep(self): | |
302 cwd = self._options.chromium | |
303 # Check for a clean workdir. | |
304 if not self.GitIsWorkdirClean(cwd=cwd): # pragma: no cover | |
305 self.Die("Workspace is not clean. Please commit or undo your changes.") | |
306 # Assert that the DEPS file is there. | |
307 if not os.path.exists(os.path.join(cwd, "DEPS")): # pragma: no cover | |
308 self.Die("DEPS file not present.") | |
309 | |
310 | |
311 class UpdateChromiumCheckout(Step): | |
312 MESSAGE = "Update the checkout and create a new branch." | |
313 | |
314 def RunStep(self): | |
315 cwd = self._options.chromium | |
316 self.GitCheckout("master", cwd=cwd) | |
317 self.GitPull(cwd=cwd) | |
318 self.GitCreateBranch(self.Config("BRANCHNAME"), cwd=cwd) | |
319 | |
320 | |
321 def ConvertToCommitNumber(step, revision): | |
322 # Simple check for git hashes. | |
323 if revision.isdigit() and len(revision) < 8: | |
324 return revision | |
325 return step.GetCommitPositionNumber( | |
326 revision, cwd=os.path.join(step._options.chromium, "v8")) | |
327 | |
328 | |
329 class RetrieveChromiumV8Releases(Step): | |
330 MESSAGE = "Retrieve V8 releases from Chromium DEPS." | |
331 | |
332 def RunStep(self): | |
333 cwd = self._options.chromium | |
334 releases = filter( | |
335 lambda r: r["branch"] in [self.vc.CandidateBranch(), | |
336 self.vc.MasterBranch()], | |
337 self["releases"]) | |
338 if not releases: # pragma: no cover | |
339 print "No releases detected. Skipping chromium history." | |
340 return True | |
341 | |
342 # Update v8 checkout in chromium. | |
343 self.GitFetchOrigin(cwd=os.path.join(cwd, "v8")) | |
344 | |
345 oldest_v8_rev = int(releases[-1]["revision"]) | |
346 | |
347 cr_releases = [] | |
348 try: | |
349 for git_hash in self.GitLog( | |
350 format="%H", grep="V8", cwd=cwd).splitlines(): | |
351 if "DEPS" not in self.GitChangedFiles(git_hash, cwd=cwd): | |
352 continue | |
353 if not self.GitCheckoutFileSafe("DEPS", git_hash, cwd=cwd): | |
354 break # pragma: no cover | |
355 deps = FileToText(os.path.join(cwd, "DEPS")) | |
356 match = DEPS_RE.search(deps) | |
357 if match: | |
358 cr_rev = self.GetCommitPositionNumber(git_hash, cwd=cwd) | |
359 if cr_rev: | |
360 v8_rev = ConvertToCommitNumber(self, match.group(1)) | |
361 cr_releases.append([cr_rev, v8_rev]) | |
362 | |
363 # Stop after reaching beyond the last v8 revision we want to update. | |
364 # We need a small buffer for possible revert/reland frenzies. | |
365 # TODO(machenbach): Subtraction is not git friendly. | |
366 if int(v8_rev) < oldest_v8_rev - 100: | |
367 break # pragma: no cover | |
368 | |
369 # Allow Ctrl-C interrupt. | |
370 except (KeyboardInterrupt, SystemExit): # pragma: no cover | |
371 pass | |
372 | |
373 # Clean up. | |
374 self.GitCheckoutFileSafe("DEPS", "HEAD", cwd=cwd) | |
375 | |
376 # Add the chromium ranges to the v8 candidates and master releases. | |
377 all_ranges = BuildRevisionRanges(cr_releases) | |
378 releases_dict = dict((r["revision"], r) for r in releases) | |
379 for revision, ranges in all_ranges.iteritems(): | |
380 releases_dict.get(revision, {})["chromium_revision"] = ranges | |
381 | |
382 | |
383 # TODO(machenbach): Unify common code with method above. | |
384 class RietrieveChromiumBranches(Step): | |
385 MESSAGE = "Retrieve Chromium branch information." | |
386 | |
387 def RunStep(self): | |
388 cwd = self._options.chromium | |
389 cand_releases = filter(lambda r: r["branch"] == self.vc.CandidateBranch(), | |
390 self["releases"]) | |
391 if not cand_releases: # pragma: no cover | |
392 print "No candidates releases detected. Skipping chromium history." | |
393 return True | |
394 | |
395 oldest_v8_rev = int(cand_releases[-1]["revision"]) | |
396 | |
397 # Filter out irrelevant branches. | |
398 branches = filter(lambda r: re.match(r"branch-heads/\d+", r), | |
399 self.GitRemotes(cwd=cwd)) | |
400 | |
401 # Transform into pure branch numbers. | |
402 branches = map(lambda r: int(re.match(r"branch-heads/(\d+)", r).group(1)), | |
403 branches) | |
404 | |
405 branches = sorted(branches, reverse=True) | |
406 | |
407 cr_branches = [] | |
408 try: | |
409 for branch in branches: | |
410 if not self.GitCheckoutFileSafe("DEPS", | |
411 "branch-heads/%d" % branch, | |
412 cwd=cwd): | |
413 break # pragma: no cover | |
414 deps = FileToText(os.path.join(cwd, "DEPS")) | |
415 match = DEPS_RE.search(deps) | |
416 if match: | |
417 v8_rev = ConvertToCommitNumber(self, match.group(1)) | |
418 cr_branches.append([str(branch), v8_rev]) | |
419 | |
420 # Stop after reaching beyond the last v8 revision we want to update. | |
421 # We need a small buffer for possible revert/reland frenzies. | |
422 # TODO(machenbach): Subtraction is not git friendly. | |
423 if int(v8_rev) < oldest_v8_rev - 100: | |
424 break # pragma: no cover | |
425 | |
426 # Allow Ctrl-C interrupt. | |
427 except (KeyboardInterrupt, SystemExit): # pragma: no cover | |
428 pass | |
429 | |
430 # Clean up. | |
431 self.GitCheckoutFileSafe("DEPS", "HEAD", cwd=cwd) | |
432 | |
433 # Add the chromium branches to the v8 candidate releases. | |
434 all_ranges = BuildRevisionRanges(cr_branches) | |
435 cand_dict = dict((r["revision"], r) for r in cand_releases) | |
436 for revision, ranges in all_ranges.iteritems(): | |
437 cand_dict.get(revision, {})["chromium_branch"] = ranges | |
438 | |
439 | |
440 class CleanUp(Step): | |
441 MESSAGE = "Clean up." | |
442 | |
443 def RunStep(self): | |
444 self.GitCheckout("master", cwd=self._options.chromium) | |
445 self.GitDeleteBranch(self.Config("BRANCHNAME"), cwd=self._options.chromium) | |
446 self.CommonCleanup() | |
447 | |
448 | |
449 class WriteOutput(Step): | |
450 MESSAGE = "Print output." | |
451 | |
452 def Run(self): | |
453 if self._options.csv: | |
454 with open(self._options.csv, "w") as f: | |
455 writer = csv.DictWriter(f, | |
456 ["version", "branch", "revision", | |
457 "chromium_revision", "patches_merged"], | |
458 restval="", | |
459 extrasaction="ignore") | |
460 for release in self["releases"]: | |
461 writer.writerow(release) | |
462 if self._options.json: | |
463 with open(self._options.json, "w") as f: | |
464 f.write(json.dumps(self["releases"])) | |
465 if not self._options.csv and not self._options.json: | |
466 print self["releases"] # pragma: no cover | |
467 | |
468 | |
469 class Releases(ScriptsBase): | |
470 def _PrepareOptions(self, parser): | |
471 parser.add_argument("-b", "--branch", default="recent", | |
472 help=("The branch to analyze. If 'all' is specified, " | |
473 "analyze all branches. If 'recent' (default) " | |
474 "is specified, track beta, stable and " | |
475 "candidates.")) | |
476 parser.add_argument("-c", "--chromium", | |
477 help=("The path to your Chromium src/ " | |
478 "directory to automate the V8 roll.")) | |
479 parser.add_argument("--csv", help="Path to a CSV file for export.") | |
480 parser.add_argument("-m", "--max-releases", type=int, default=0, | |
481 help="The maximum number of releases to track.") | |
482 parser.add_argument("--json", help="Path to a JSON file for export.") | |
483 | |
484 def _ProcessOptions(self, options): # pragma: no cover | |
485 return True | |
486 | |
487 def _Config(self): | |
488 return { | |
489 "BRANCHNAME": "retrieve-v8-releases", | |
490 "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile", | |
491 } | |
492 | |
493 def _Steps(self): | |
494 return [ | |
495 Preparation, | |
496 RetrieveV8Releases, | |
497 SwitchChromium, | |
498 UpdateChromiumCheckout, | |
499 RetrieveChromiumV8Releases, | |
500 RietrieveChromiumBranches, | |
501 CleanUp, | |
502 WriteOutput, | |
503 ] | |
504 | |
505 | |
506 if __name__ == "__main__": # pragma: no cover | |
507 sys.exit(Releases().Run()) | |
OLD | NEW |