OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2016 The LUCI Authors. All rights reserved. | 2 # Copyright 2016 The LUCI Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
5 | 5 |
6 import base64 | 6 import base64 |
7 import itertools | 7 import itertools |
8 import json | 8 import json |
9 import unittest | 9 import unittest |
10 | 10 |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
63 self.assertListEqual(list(real_args), full_args) | 63 self.assertListEqual(list(real_args), full_args) |
64 raise data_or_exception | 64 raise data_or_exception |
65 else: | 65 else: |
66 def _inner(*real_args): | 66 def _inner(*real_args): |
67 self.assertListEqual(list(real_args), full_args) | 67 self.assertListEqual(list(real_args), full_args) |
68 return data_or_exception | 68 return data_or_exception |
69 return _inner | 69 return _inner |
70 | 70 |
71 def g_metadata_calls(self, dirname='dir', commit='a'*40, | 71 def g_metadata_calls(self, dirname='dir', commit='a'*40, |
72 email='foo@example.com', msg='hello\nworld', | 72 email='foo@example.com', msg='hello\nworld', |
73 commit_timestamp=1492131405, config=None): | 73 commit_timestamp=1492131405, config=None, |
| 74 diff=('foo', 'bar')): |
74 config = config or {'api_version': 2} | 75 config = config or {'api_version': 2} |
75 | 76 |
76 return [ | 77 return [ |
77 self.g([ | 78 self.g([ |
78 '-C', dirname, 'show', '-s', '--format=%aE%n%ct%n%B', commit | 79 '-C', dirname, 'show', '-s', '--format=%aE%n%ct%n%B', commit |
79 ], '%s\n%d\n%s\n' % (email, commit_timestamp, msg)), | 80 ], '%s\n%d\n%s\n' % (email, commit_timestamp, msg)), |
80 self.g([ | 81 self.g([ |
81 '-C', dirname, 'cat-file', 'blob', commit+':infra/config/recipes.cfg' | 82 '-C', dirname, 'cat-file', 'blob', commit+':infra/config/recipes.cfg' |
82 ], json.dumps(config)) | 83 ], json.dumps(config)), |
| 84 self.g([ |
| 85 '-C', dirname, |
| 86 'diff-tree', '-r', '--no-commit-id', '--name-only', commit+'^!', |
| 87 ], '\n'.join(diff)) |
83 ] | 88 ] |
84 | 89 |
85 @mock.patch('os.path.isdir') | 90 @mock.patch('os.path.isdir') |
86 @mock.patch('recipe_engine.fetch.GitBackend._execute') | 91 @mock.patch('recipe_engine.fetch.GitBackend._execute') |
87 def test_fresh_clone(self, git, isdir): | 92 def test_fresh_clone(self, git, isdir): |
88 isdir.return_value = False | 93 isdir.return_value = False |
89 git.side_effect = multi(*([ | 94 git.side_effect = multi(*([ |
90 self.g(['init', 'dir']), | 95 self.g(['init', 'dir']), |
91 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), | 96 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), |
92 ] + self.g_metadata_calls() + [ | 97 ] + self.g_metadata_calls() + [ |
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
190 self.g(['init', 'dir']), | 195 self.g(['init', 'dir']), |
191 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), | 196 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), |
192 ] + self.g_metadata_calls())) | 197 ] + self.g_metadata_calls())) |
193 | 198 |
194 result = fetch.GitBackend('dir', 'repo', True).commit_metadata('revision') | 199 result = fetch.GitBackend('dir', 'repo', True).commit_metadata('revision') |
195 self.assertEqual(result, fetch.CommitMetadata( | 200 self.assertEqual(result, fetch.CommitMetadata( |
196 revision = 'a'*40, | 201 revision = 'a'*40, |
197 author_email = 'foo@example.com', | 202 author_email = 'foo@example.com', |
198 commit_timestamp = 1492131405, | 203 commit_timestamp = 1492131405, |
199 message_lines = ('hello', 'world'), | 204 message_lines = ('hello', 'world'), |
200 spec = package_pb2.Package(api_version=2) | 205 spec = package_pb2.Package(api_version=2), |
| 206 roll_candidate = True, |
201 )) | 207 )) |
202 self.assertMultiDone(git) | 208 self.assertMultiDone(git) |
203 | 209 |
204 | 210 |
205 class TestGitiles(unittest.TestCase): | 211 class TestGitiles(unittest.TestCase): |
206 def setUp(self): | 212 def setUp(self): |
207 requests_ssl.disable_check() | 213 requests_ssl.disable_check() |
208 fetch.Backend._GIT_METADATA_CACHE = {} | 214 fetch.Backend._GIT_METADATA_CACHE = {} |
209 fetch.GitilesBackend._COMMIT_JSON_CACHE = {} | 215 fetch.GitilesBackend._COMMIT_JSON_CACHE = {} |
210 | 216 |
211 self.proto_text = u"""{ | 217 self.proto_text = u"""{ |
212 "api_version": 2, | 218 "api_version": 2, |
213 "project_id": "foo", | 219 "project_id": "foo", |
214 "recipes_path": "path/to/recipes" | 220 "recipes_path": "path/to/recipes" |
215 }""".lstrip() | 221 }""".lstrip() |
216 | 222 |
217 self.a = 'a'*40 | 223 self.a = 'a'*40 |
218 self.a_dat = { | 224 self.a_dat = { |
219 'commit': self.a, | 225 'commit': self.a, |
220 'author': {'email': 'foo@example.com'}, | 226 'author': {'email': 'foo@example.com'}, |
221 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, | 227 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, |
222 'message': 'message', | 228 'message': 'message', |
| 229 'tree_diff': [ |
| 230 { |
| 231 'old_path': 'unrelated', |
| 232 'new_path': 'unrelated', |
| 233 }, |
| 234 { |
| 235 'old_path': 'path/to/recipes/foo', |
| 236 'new_path': 'path/to/recipes/bar', |
| 237 }, |
| 238 ] |
223 } | 239 } |
224 | 240 |
225 self.a_meta = fetch.CommitMetadata( | 241 self.a_meta = fetch.CommitMetadata( |
226 revision = 'a'*40, | 242 revision = 'a'*40, |
227 author_email = 'foo@example.com', | 243 author_email = 'foo@example.com', |
228 commit_timestamp = 1492131405, | 244 commit_timestamp = 1492131405, |
229 message_lines = ('message',), | 245 message_lines = ('message',), |
230 spec = package_pb2.Package( | 246 spec = package_pb2.Package( |
231 api_version = 2, | 247 api_version = 2, |
232 project_id = 'foo', | 248 project_id = 'foo', |
233 recipes_path = 'path/to/recipes', | 249 recipes_path = 'path/to/recipes', |
234 ) | 250 ), |
| 251 roll_candidate = True, |
235 ) | 252 ) |
236 | 253 |
237 def assertMultiDone(self, mocked_call): | 254 def assertMultiDone(self, mocked_call): |
238 with self.assertRaises(NoMoreExpectatedCalls): | 255 with self.assertRaises(NoMoreExpectatedCalls): |
239 mocked_call() | 256 mocked_call() |
240 | 257 |
241 def j(self, url, data, status_code=200): | 258 def j(self, url, data, status_code=200): |
242 """Mock a request.get to return json data.""" | 259 """Mock a request.get to return json data.""" |
243 return self.r(url, u')]}\'\n'+json.dumps(data), status_code) | 260 return self.r(url, u')]}\'\n'+json.dumps(data), status_code) |
244 | 261 |
(...skipping 20 matching lines...) Expand all Loading... |
265 status_code=status_code) | 282 status_code=status_code) |
266 return _inner | 283 return _inner |
267 | 284 |
268 @mock.patch('__builtin__.open', mock.mock_open()) | 285 @mock.patch('__builtin__.open', mock.mock_open()) |
269 @mock.patch('shutil.rmtree') | 286 @mock.patch('shutil.rmtree') |
270 @mock.patch('os.makedirs') | 287 @mock.patch('os.makedirs') |
271 @mock.patch('tarfile.open') | 288 @mock.patch('tarfile.open') |
272 @mock.patch('requests.get') | 289 @mock.patch('requests.get') |
273 def test_checkout(self, requests_get, _tarfile_open, makedirs, rmtree): | 290 def test_checkout(self, requests_get, _tarfile_open, makedirs, rmtree): |
274 requests_get.side_effect = multi( | 291 requests_get.side_effect = multi( |
275 self.j('repo/+/revision?format=JSON', self.a_dat), | 292 self.j('repo/+/revision?name-status=1&format=JSON', self.a_dat), |
276 self.j('repo/+/%s?format=JSON' % self.a, self.a_dat), | 293 self.j('repo/+/%s?name-status=1&format=JSON' % self.a, self.a_dat), |
277 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, | 294 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, |
278 self.proto_text), | 295 self.proto_text), |
279 self.d('repo/+archive/%s/path/to/recipes.tar.gz' % self.a, ''), | 296 self.d('repo/+archive/%s/path/to/recipes.tar.gz' % self.a, ''), |
280 ) | 297 ) |
281 | 298 |
282 fetch.GitilesBackend('dir', 'repo', True).checkout('revision') | 299 fetch.GitilesBackend('dir', 'repo', True).checkout('revision') |
283 | 300 |
284 makedirs.assert_has_calls([ | 301 makedirs.assert_has_calls([ |
285 mock.call('dir/infra/config'), | 302 mock.call('dir/infra/config'), |
286 mock.call('dir/path/to/recipes'), | 303 mock.call('dir/path/to/recipes'), |
287 ]) | 304 ]) |
288 | 305 |
289 rmtree.assert_called_once_with('dir', ignore_errors=True) | 306 rmtree.assert_called_once_with('dir', ignore_errors=True) |
290 self.assertMultiDone(requests_get) | 307 self.assertMultiDone(requests_get) |
291 | 308 |
292 @mock.patch('requests.get') | 309 @mock.patch('requests.get') |
293 def test_updates(self, requests_get): | 310 def test_updates(self, requests_get): |
294 sha_a = 'a'*40 | 311 sha_a = 'a'*40 |
295 sha_b = 'b'*40 | 312 sha_b = 'b'*40 |
296 log_json = { | 313 log_json = { |
297 'log': [ | 314 'log': [ |
298 { | 315 { |
299 'commit': sha_b, | 316 'commit': sha_b, |
300 'author': {'email': 'foo@example.com'}, | 317 'author': {'email': 'foo@example.com'}, |
301 'committer': {'time': 'Fri Apr 14 00:58:45 2017'}, | 318 'committer': {'time': 'Fri Apr 14 00:58:45 2017'}, |
302 'message': 'message', | 319 'message': 'message', |
303 'tree_diff': [ | 320 'tree_diff': [ |
304 { | 321 { |
305 'old_path': '/dev/null', | 322 'old_path': '/dev/null', |
306 'new_path': 'path1/foo', | 323 'new_path': 'path/to/recipes/path1/foo', |
307 }, | 324 }, |
308 ], | 325 ], |
309 }, | 326 }, |
310 { | 327 { |
311 'commit': 'def456', | 328 'commit': 'def456', |
312 'author': {'email': 'foo@example.com'}, | 329 'author': {'email': 'foo@example.com'}, |
313 'committer': {'time': 'Fri Apr 14 00:57:45 2017'}, | 330 'committer': {'time': 'Fri Apr 14 00:57:45 2017'}, |
314 'message': 'message', | 331 'message': 'message', |
315 'tree_diff': [ | 332 'tree_diff': [ |
316 { | 333 { |
317 'old_path': '/dev/null', | 334 'old_path': '/dev/null', |
318 'new_path': 'path8/foo', | 335 'new_path': 'path8/foo', |
319 }, | 336 }, |
320 ], | 337 ], |
321 }, | 338 }, |
322 { | 339 { |
323 'commit': sha_a, | 340 'commit': sha_a, |
324 'author': {'email': 'foo@example.com'}, | 341 'author': {'email': 'foo@example.com'}, |
325 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, | 342 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, |
326 'message': 'message', | 343 'message': 'message', |
327 'tree_diff': [ | 344 'tree_diff': [ |
328 { | 345 { |
329 'old_path': '/dev/null', | 346 'old_path': '/dev/null', |
330 'new_path': 'path8/foo', | 347 'new_path': 'path/to/recipes/path8/foo', |
331 }, | 348 }, |
332 { | 349 { |
333 'old_path': 'path2/foo', | 350 'old_path': 'path2/foo', |
334 'new_path': '/dev/null', | 351 'new_path': '/dev/null', |
335 }, | 352 }, |
336 ], | 353 ], |
337 }, | 354 }, |
338 ], | 355 ], |
339 } | 356 } |
340 | 357 |
341 requests_get.side_effect = multi( | 358 requests_get.side_effect = multi( |
342 self.j('repo/+/reva?format=JSON', { | 359 self.j('repo/+/reva?name-status=1&format=JSON', log_json['log'][2]), |
343 'commit': sha_a, | 360 self.j('repo/+/revb?name-status=1&format=JSON', log_json['log'][0]), |
344 'author': {'email': 'foo@example.com'}, | |
345 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, | |
346 'message': 'message', | |
347 }), | |
348 self.j('repo/+/revb?format=JSON', { | |
349 'commit': sha_b, | |
350 'author': {'email': 'foo@example.com'}, | |
351 'committer': {'time': 'Fri Apr 14 00:58:45 2017'}, | |
352 'message': 'message', | |
353 }), | |
354 self.j('repo/+log/%s..%s?name-status=1&format=JSON' % (sha_a, sha_b), | 361 self.j('repo/+log/%s..%s?name-status=1&format=JSON' % (sha_a, sha_b), |
355 log_json), | 362 log_json), |
356 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_a, | 363 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_a, |
357 self.proto_text), | 364 self.proto_text), |
| 365 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % 'def456', |
| 366 self.proto_text), |
358 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_b, | 367 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_b, |
359 self.proto_text), | 368 self.proto_text), |
360 ) | 369 ) |
361 | 370 |
362 be = fetch.GitilesBackend('dir', 'repo', True) | 371 be = fetch.GitilesBackend('dir', 'repo', True) |
363 self.assertEqual(sha_a, be.resolve_refspec('reva')) | 372 self.assertEqual(sha_a, be.resolve_refspec('reva')) |
364 self.assertEqual(sha_b, be.resolve_refspec('revb')) | 373 self.assertEqual(sha_b, be.resolve_refspec('revb')) |
365 | 374 |
366 self.assertEqual( | 375 self.assertEqual( |
367 [self.a_meta, | 376 [self.a_meta, |
368 fetch.CommitMetadata( | 377 fetch.CommitMetadata( |
| 378 revision = 'def456', |
| 379 author_email = 'foo@example.com', |
| 380 commit_timestamp = 1492131465, |
| 381 message_lines = ('message',), |
| 382 spec = package_pb2.Package( |
| 383 api_version = 2, |
| 384 project_id = 'foo', |
| 385 recipes_path = 'path/to/recipes', |
| 386 ), |
| 387 roll_candidate = False, |
| 388 ), |
| 389 fetch.CommitMetadata( |
369 revision = sha_b, | 390 revision = sha_b, |
370 author_email = 'foo@example.com', | 391 author_email = 'foo@example.com', |
371 commit_timestamp = 1492131525, | 392 commit_timestamp = 1492131525, |
372 message_lines = ('message',), | 393 message_lines = ('message',), |
373 spec = package_pb2.Package( | 394 spec = package_pb2.Package( |
374 api_version = 2, | 395 api_version = 2, |
375 project_id = 'foo', | 396 project_id = 'foo', |
376 recipes_path = 'path/to/recipes', | 397 recipes_path = 'path/to/recipes', |
377 ) | 398 ), |
| 399 roll_candidate = True, |
378 )], | 400 )], |
379 be.updates(sha_a, sha_b, ['path1', 'path2'])) | 401 be.updates(sha_a, sha_b)) |
380 self.assertMultiDone(requests_get) | 402 self.assertMultiDone(requests_get) |
381 | 403 |
382 | 404 |
383 @mock.patch('requests.get') | 405 @mock.patch('requests.get') |
384 def test_commit_metadata(self, requests_get): | 406 def test_commit_metadata(self, requests_get): |
385 requests_get.side_effect = multi( | 407 requests_get.side_effect = multi( |
386 self.j('repo/+/revision?format=JSON', self.a_dat), | 408 self.j('repo/+/revision?name-status=1&format=JSON', self.a_dat), |
387 self.j('repo/+/%s?format=JSON' % self.a, self.a_dat), | 409 self.j('repo/+/%s?name-status=1&format=JSON' % self.a, self.a_dat), |
388 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, | 410 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, |
389 self.proto_text) | 411 self.proto_text) |
390 ) | 412 ) |
391 | 413 |
392 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( | 414 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( |
393 'revision') | 415 'revision') |
394 self.assertEqual(result, self.a_meta) | 416 self.assertEqual(result, self.a_meta) |
395 self.assertMultiDone(requests_get) | 417 self.assertMultiDone(requests_get) |
396 | 418 |
397 @mock.patch('requests.get') | 419 @mock.patch('requests.get') |
398 def test_non_transient_error(self, requests_get): | 420 def test_non_transient_error(self, requests_get): |
399 requests_get.side_effect = multi( | 421 requests_get.side_effect = multi( |
400 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(403, '')), | 422 self.r('repo/+/revision?name-status=1&format=JSON', |
| 423 fetch.GitilesFetchError(403, '')), |
401 ) | 424 ) |
402 with self.assertRaises(fetch.GitilesFetchError): | 425 with self.assertRaises(fetch.GitilesFetchError): |
403 fetch.GitilesBackend('dir', 'repo', True).commit_metadata( | 426 fetch.GitilesBackend('dir', 'repo', True).commit_metadata( |
404 'revision') | 427 'revision') |
405 self.assertMultiDone(requests_get) | 428 self.assertMultiDone(requests_get) |
406 | 429 |
407 @mock.patch('requests.get') | 430 @mock.patch('requests.get') |
408 @mock.patch('time.sleep') | 431 @mock.patch('time.sleep') |
409 @mock.patch('logging.exception') | 432 @mock.patch('logging.exception') |
410 def test_transient_retry(self, _logging_exception, _time_sleep, requests_get): | 433 def test_transient_retry(self, _logging_exception, _time_sleep, requests_get): |
411 requests_get.side_effect = multi( | 434 requests_get.side_effect = multi( |
412 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), | 435 self.r('repo/+/revision?name-status=1&format=JSON', |
413 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), | 436 fetch.GitilesFetchError(500, '')), |
414 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), | 437 self.r('repo/+/revision?name-status=1&format=JSON', |
415 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), | 438 fetch.GitilesFetchError(500, '')), |
416 self.j('repo/+/revision?format=JSON', self.a_dat), | 439 self.r('repo/+/revision?name-status=1&format=JSON', |
417 self.j('repo/+/%s?format=JSON' % self.a, self.a_dat), | 440 fetch.GitilesFetchError(500, '')), |
| 441 self.r('repo/+/revision?name-status=1&format=JSON', |
| 442 fetch.GitilesFetchError(500, '')), |
| 443 self.j('repo/+/revision?name-status=1&format=JSON', self.a_dat), |
| 444 self.j('repo/+/%s?name-status=1&format=JSON' % self.a, self.a_dat), |
418 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, | 445 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, |
419 self.proto_text), | 446 self.proto_text), |
420 ) | 447 ) |
421 | 448 |
422 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( | 449 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( |
423 'revision') | 450 'revision') |
424 self.assertEqual(result, self.a_meta) | 451 self.assertEqual(result, self.a_meta) |
425 self.assertMultiDone(requests_get) | 452 self.assertMultiDone(requests_get) |
426 | 453 |
427 if __name__ == '__main__': | 454 if __name__ == '__main__': |
428 unittest.main() | 455 unittest.main() |
OLD | NEW |