OLD | NEW |
| (Empty) |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 import collections | |
6 | |
7 from dashboard.common import namespaced_stored_object | |
8 from dashboard.services import gitiles_service | |
9 | |
10 | |
11 _REPOSITORIES_KEY = 'repositories' | |
12 | |
13 | |
14 class NonLinearError(Exception): | |
15 """Raised when trying to find the midpoint of Changes that are not linear.""" | |
16 | |
17 | |
18 class Change(collections.namedtuple('Change', | |
19 ('base_commit', 'deps', 'patch'))): | |
20 """A particular set of Deps with or without an additional patch applied. | |
21 | |
22 For example, a Change might sync to src@9064a40 and catapult@8f26966, | |
23 then apply patch 2423293002. | |
24 """ | |
25 | |
26 def __new__(cls, base_commit, deps=frozenset(), patch=None): | |
27 """Create a Change. | |
28 | |
29 Args: | |
30 base_commit: A Dep representing the initial commit to sync to. The DEPS | |
31 file at that commit implies the default commits for any dependencies. | |
32 deps: An optional iterable of Deps to override the dependencies implied | |
33 by base_commit. | |
34 patch: An optional Patch to apply to the Change. | |
35 """ | |
36 return super(Change, cls).__new__(cls, base_commit, frozenset(deps), patch) | |
37 | |
38 def __str__(self): | |
39 string = ' '.join(str(dep) for dep in self.all_deps) | |
40 if self.patch: | |
41 string += ' + ' + str(self.patch) | |
42 return string | |
43 | |
44 @property | |
45 def id_string(self): | |
46 string = ' '.join(dep.id_string for dep in self.all_deps) | |
47 if self.patch: | |
48 string += ' + ' + self.patch.id_string | |
49 return string | |
50 | |
51 @property | |
52 def all_deps(self): | |
53 return tuple([self.base_commit] + sorted(self.deps)) | |
54 | |
55 def AsDict(self): | |
56 return { | |
57 'base_commit': self.base_commit.AsDict(), | |
58 'deps': [dep.AsDict() for dep in sorted(self.deps)], | |
59 'patch': self.patch.AsDict() if self.patch else None, | |
60 } | |
61 | |
62 @classmethod | |
63 def FromDict(cls, data): | |
64 base_commit = Dep.FromDict(data['base_commit']) | |
65 | |
66 kwargs = {} | |
67 if 'deps' in data: | |
68 kwargs['deps'] = tuple(Dep.FromDict(dep) for dep in data['deps']) | |
69 if 'patch' in data: | |
70 kwargs['patch'] = Patch.FromDict(data['patch']) | |
71 | |
72 return cls(base_commit, **kwargs) | |
73 | |
74 @classmethod | |
75 def Midpoint(cls, change_a, change_b): | |
76 """Return a Change halfway between the two given Changes. | |
77 | |
78 A NonLinearError is raised if the Changes are not linear. The Changes are | |
79 not linear if any of the following is true: | |
80 * They have different base repositories. | |
81 * They have different patches. | |
82 * Their repositories differ even after expanding DEPS rolls. | |
83 See change_test.py for examples of linear and nonlinear Changes. | |
84 | |
85 The behavior is undefined if either of the Changes have multiple Deps with | |
86 the same repository. | |
87 | |
88 Args: | |
89 change_a: The first Change in the range. | |
90 change_b: The last Change in the range. | |
91 | |
92 Returns: | |
93 A new Change representing the midpoint. | |
94 The commit before the midpoint if the range has an even number of commits. | |
95 None if the range is empty, or the Changes are given in the wrong order. | |
96 | |
97 Raises: | |
98 NonLinearError: The Changes are not linear. | |
99 """ | |
100 if change_a.base_commit.repository != change_b.base_commit.repository: | |
101 raise NonLinearError( | |
102 'Change A has base repo "%s" and Change B has base repo "%s".' % | |
103 (change_a.base_commit.repository, change_b.base_commit.repository)) | |
104 | |
105 if change_a.patch != change_b.patch: | |
106 raise NonLinearError( | |
107 'Change A has patch "%s" and Change B has patch "%s".' % | |
108 (change_a.patch, change_b.patch)) | |
109 | |
110 if change_a == change_b: | |
111 return None | |
112 | |
113 # Find the midpoint of every pair of Deps, expanding DEPS rolls as we go. | |
114 midpoint_deps = {} | |
115 | |
116 repositories_a = {dep.repository: dep for dep in change_a.all_deps} | |
117 repositories_b = {dep.repository: dep for dep in change_b.all_deps} | |
118 | |
119 # Match up all the Deps by repository. | |
120 while frozenset(repositories_a.iterkeys()).intersection( | |
121 frozenset(repositories_b.iterkeys())): | |
122 # Choose an arbitrary pair of Deps with the same repository. | |
123 shared_repositories = set(repositories_a.iterkeys()).intersection( | |
124 set(repositories_b.iterkeys())) | |
125 repository = shared_repositories.pop() | |
126 dep_a = repositories_a.pop(repository) | |
127 dep_b = repositories_b.pop(repository) | |
128 | |
129 if dep_a == dep_b: | |
130 # The Deps are the same. | |
131 midpoint_deps[repository] = dep_a | |
132 continue | |
133 | |
134 midpoint_dep = Dep.Midpoint(dep_a, dep_b) | |
135 if midpoint_dep: | |
136 # The Deps are not adjacent. | |
137 midpoint_deps[repository] = midpoint_dep | |
138 continue | |
139 | |
140 # The Deps are adjacent. Figure out if it's a DEPS roll. | |
141 deps_a = dep_a.Deps() | |
142 deps_b = dep_b.Deps() | |
143 if deps_a == deps_b: | |
144 # Not a DEPS roll. The Changes really are adjacent. | |
145 return None | |
146 | |
147 # DEPS roll! Expand the roll. | |
148 for dep in deps_a.difference(deps_b): | |
149 if dep.repository in midpoint_deps: | |
150 raise NonLinearError('Tried to take the midpoint across a DEPS roll, ' | |
151 'but the underlying Dep is already overriden in ' | |
152 'both Changes.') | |
153 if dep.repository not in repositories_a: | |
154 repositories_a[dep.repository] = dep | |
155 for dep in deps_b.difference(deps_a): | |
156 if dep.repository in midpoint_deps: | |
157 raise NonLinearError('Tried to take the midpoint across a DEPS roll, ' | |
158 'but the underlying Dep is already overriden in ' | |
159 'both Changes.') | |
160 if dep.repository not in repositories_b: | |
161 repositories_b[dep.repository] = dep | |
162 midpoint_deps[repository] = dep_a | |
163 | |
164 # Now that the DEPS are expanded, check to see if the repositories differ. | |
165 if repositories_a or repositories_b: | |
166 raise NonLinearError( | |
167 'Repositories differ between Change A and Change B: %s' % | |
168 ', '.join(sorted(repositories_a.keys() + repositories_b.keys()))) | |
169 | |
170 # Create our new Change! | |
171 base_commit = midpoint_deps.pop(change_a.base_commit.repository) | |
172 return cls(base_commit, midpoint_deps.itervalues(), change_a.patch) | |
173 | |
174 | |
175 class Dep(collections.namedtuple('Dep', ('repository', 'git_hash'))): | |
176 """A git repository pinned to a particular commit.""" | |
177 | |
178 def __str__(self): | |
179 return self.repository + '@' + self.git_hash[:7] | |
180 | |
181 @property | |
182 def id_string(self): | |
183 return self.repository + '@' + self.git_hash | |
184 | |
185 @property | |
186 def repository_url(self): | |
187 """The HTTPS URL of the repository as passed to `git clone`.""" | |
188 repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY) | |
189 return repositories[self.repository]['repository_url'] | |
190 | |
191 def Deps(self): | |
192 """Return the DEPS of this Dep as a frozenset of Deps.""" | |
193 # Download and execute DEPS file. | |
194 deps_file_contents = gitiles_service.FileContents( | |
195 self.repository_url, self.git_hash, 'DEPS') | |
196 deps_data = {'Var': lambda variable: deps_data['vars'][variable]} | |
197 exec deps_file_contents in deps_data # pylint: disable=exec-used | |
198 | |
199 # Pull out deps dict, including OS-specific deps. | |
200 deps_dict = deps_data['deps'] | |
201 for deps_os in deps_data.get('deps_os', {}).itervalues(): | |
202 deps_dict.update(deps_os) | |
203 | |
204 # Convert deps strings to Dep objects. | |
205 deps = [] | |
206 for dep_string in deps_dict.itervalues(): | |
207 dep_string_parts = dep_string.split('@') | |
208 if len(dep_string_parts) < 2: | |
209 continue # Dep is not pinned to any particular revision. | |
210 if len(dep_string_parts) > 2: | |
211 raise NotImplementedError('Unknown DEP format: ' + dep_string) | |
212 | |
213 repository_url, git_hash = dep_string_parts | |
214 repository = _Repository(repository_url) | |
215 if not repository: | |
216 _AddRepository(repository_url) | |
217 repository = _Repository(repository_url) | |
218 deps.append(Dep(repository, git_hash)) | |
219 | |
220 return frozenset(deps) | |
221 | |
222 def AsDict(self): | |
223 return { | |
224 'repository': self.repository, | |
225 'git_hash': self.git_hash, | |
226 'url': self.repository_url + '/+/' + self.git_hash, | |
227 } | |
228 | |
229 @classmethod | |
230 def FromDict(cls, data): | |
231 """Create a Dep from a dict. | |
232 | |
233 If the repository is a repository URL, it will be translated to its short | |
234 form name. | |
235 | |
236 Raises: | |
237 KeyError: The repository name is not in the local datastore, | |
238 or the git hash is not valid. | |
239 """ | |
240 repository = data['repository'] | |
241 | |
242 # Translate repository if it's a URL. | |
243 repository_from_url = _Repository(repository) | |
244 if repository_from_url: | |
245 repository = repository_from_url | |
246 | |
247 dep = cls(repository, data['git_hash']) | |
248 | |
249 try: | |
250 gitiles_service.CommitInfo(dep.repository_url, dep.git_hash) | |
251 except gitiles_service.NotFoundError as e: | |
252 raise KeyError(str(e)) | |
253 | |
254 return dep | |
255 | |
256 @classmethod | |
257 def Midpoint(cls, dep_a, dep_b): | |
258 """Return a Dep halfway between the two given Deps. | |
259 | |
260 Uses Gitiles to look up the commit range. | |
261 | |
262 Args: | |
263 dep_a: The first Dep in the range. | |
264 dep_b: The last Dep in the range. | |
265 | |
266 Returns: | |
267 A new Dep representing the midpoint. | |
268 The commit before the midpoint if the range has an even number of commits. | |
269 None if the range is empty, or the Deps are given in the wrong order. | |
270 | |
271 Raises: | |
272 ValueError: The Deps are in different repositories. | |
273 """ | |
274 if dep_a.repository != dep_b.repository: | |
275 raise ValueError("Can't find the midpoint of Deps in differing " | |
276 'repositories: "%s" and "%s"' % (dep_a, dep_b)) | |
277 | |
278 commits = gitiles_service.CommitRange(dep_a.repository_url, | |
279 dep_a.git_hash, dep_b.git_hash) | |
280 # We don't handle NotFoundErrors because we assume that all Deps either came | |
281 # from this method or were already validated elsewhere. | |
282 if len(commits) <= 1: | |
283 return None | |
284 commits = commits[1:] # Remove dep_b from the range. | |
285 | |
286 return cls(dep_a.repository, commits[len(commits) / 2]['commit']) | |
287 | |
288 | |
289 class Patch(collections.namedtuple('Patch', ('server', 'issue', 'patchset'))): | |
290 """A patch in Rietveld.""" | |
291 # TODO: Support Gerrit. | |
292 # https://github.com/catapult-project/catapult/issues/3599 | |
293 | |
294 def __str__(self): | |
295 return self.id_string | |
296 | |
297 @property | |
298 def id_string(self): | |
299 return '%s/%d/%d' % (self.server, self.issue, self.patchset) | |
300 | |
301 def AsDict(self): | |
302 return self._asdict() | |
303 | |
304 @classmethod | |
305 def FromDict(cls, data): | |
306 # TODO: Validate to ensure the patch exists on the server. | |
307 return cls(data['server'], data['issue'], data['patchset']) | |
308 | |
309 | |
310 def _Repository(repository_url): | |
311 if repository_url.endswith('.git'): | |
312 repository_url = repository_url[:-4] | |
313 | |
314 repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY) | |
315 for repo_label, repo_info in repositories.iteritems(): | |
316 if repository_url == repo_info['repository_url']: | |
317 return repo_label | |
318 | |
319 return None | |
320 | |
321 | |
322 def _AddRepository(repository_url): | |
323 if repository_url.endswith('.git'): | |
324 repository_url = repository_url[:-4] | |
325 | |
326 repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY) | |
327 repository = repository_url.split('/')[-1] | |
328 | |
329 if repository in repositories: | |
330 raise AssertionError("Attempted to add a repository that's already in the " | |
331 'Datastore: %s: %s' % (repository, repository_url)) | |
332 | |
333 repositories[repository] = {'repository_url': repository_url} | |
334 namespaced_stored_object.Set(_REPOSITORIES_KEY, repositories) | |
OLD | NEW |