OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2016 The LUCI Authors. All rights reserved. | 2 # Copyright 2016 The LUCI Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
5 | 5 |
6 import base64 | 6 import base64 |
7 import itertools | 7 import itertools |
8 import json | 8 import json |
9 import unittest | 9 import unittest |
10 | 10 |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
63 self.assertListEqual(list(real_args), full_args) | 63 self.assertListEqual(list(real_args), full_args) |
64 raise data_or_exception | 64 raise data_or_exception |
65 else: | 65 else: |
66 def _inner(*real_args): | 66 def _inner(*real_args): |
67 self.assertListEqual(list(real_args), full_args) | 67 self.assertListEqual(list(real_args), full_args) |
68 return data_or_exception | 68 return data_or_exception |
69 return _inner | 69 return _inner |
70 | 70 |
71 def g_metadata_calls(self, dirname='dir', commit='a'*40, | 71 def g_metadata_calls(self, dirname='dir', commit='a'*40, |
72 email='foo@example.com', msg='hello\nworld', | 72 email='foo@example.com', msg='hello\nworld', |
73 commit_timestamp=1492131405, config=None, | 73 commit_timestamp=1492131405, config=None): |
74 diff=('foo', 'bar')): | |
75 config = config or {'api_version': 2} | 74 config = config or {'api_version': 2} |
76 | 75 |
77 return [ | 76 return [ |
78 self.g([ | 77 self.g([ |
79 '-C', dirname, 'show', '-s', '--format=%aE%n%ct%n%B', commit | 78 '-C', dirname, 'show', '-s', '--format=%aE%n%ct%n%B', commit |
80 ], '%s\n%d\n%s\n' % (email, commit_timestamp, msg)), | 79 ], '%s\n%d\n%s\n' % (email, commit_timestamp, msg)), |
81 self.g([ | 80 self.g([ |
82 '-C', dirname, 'cat-file', 'blob', commit+':infra/config/recipes.cfg' | 81 '-C', dirname, 'cat-file', 'blob', commit+':infra/config/recipes.cfg' |
83 ], json.dumps(config)), | 82 ], json.dumps(config)) |
84 self.g([ | |
85 '-C', dirname, | |
86 'diff-tree', '-r', '--no-commit-id', '--name-only', commit+'^!', | |
87 ], '\n'.join(diff)) | |
88 ] | 83 ] |
89 | 84 |
90 @mock.patch('os.path.isdir') | 85 @mock.patch('os.path.isdir') |
91 @mock.patch('recipe_engine.fetch.GitBackend._execute') | 86 @mock.patch('recipe_engine.fetch.GitBackend._execute') |
92 def test_fresh_clone(self, git, isdir): | 87 def test_fresh_clone(self, git, isdir): |
93 isdir.return_value = False | 88 isdir.return_value = False |
94 git.side_effect = multi(*([ | 89 git.side_effect = multi(*([ |
95 self.g(['init', 'dir']), | 90 self.g(['init', 'dir']), |
96 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), | 91 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), |
97 ] + self.g_metadata_calls() + [ | 92 ] + self.g_metadata_calls() + [ |
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
195 self.g(['init', 'dir']), | 190 self.g(['init', 'dir']), |
196 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), | 191 self.g(['-C', 'dir', 'ls-remote', 'repo', 'revision'], 'a'*40), |
197 ] + self.g_metadata_calls())) | 192 ] + self.g_metadata_calls())) |
198 | 193 |
199 result = fetch.GitBackend('dir', 'repo', True).commit_metadata('revision') | 194 result = fetch.GitBackend('dir', 'repo', True).commit_metadata('revision') |
200 self.assertEqual(result, fetch.CommitMetadata( | 195 self.assertEqual(result, fetch.CommitMetadata( |
201 revision = 'a'*40, | 196 revision = 'a'*40, |
202 author_email = 'foo@example.com', | 197 author_email = 'foo@example.com', |
203 commit_timestamp = 1492131405, | 198 commit_timestamp = 1492131405, |
204 message_lines = ('hello', 'world'), | 199 message_lines = ('hello', 'world'), |
205 spec = package_pb2.Package(api_version=2), | 200 spec = package_pb2.Package(api_version=2) |
206 roll_candidate = True, | |
207 )) | 201 )) |
208 self.assertMultiDone(git) | 202 self.assertMultiDone(git) |
209 | 203 |
210 | 204 |
211 class TestGitiles(unittest.TestCase): | 205 class TestGitiles(unittest.TestCase): |
212 def setUp(self): | 206 def setUp(self): |
213 requests_ssl.disable_check() | 207 requests_ssl.disable_check() |
214 fetch.Backend._GIT_METADATA_CACHE = {} | 208 fetch.Backend._GIT_METADATA_CACHE = {} |
215 fetch.GitilesBackend._COMMIT_JSON_CACHE = {} | 209 fetch.GitilesBackend._COMMIT_JSON_CACHE = {} |
216 | 210 |
217 self.proto_text = u"""{ | 211 self.proto_text = u"""{ |
218 "api_version": 2, | 212 "api_version": 2, |
219 "project_id": "foo", | 213 "project_id": "foo", |
220 "recipes_path": "path/to/recipes" | 214 "recipes_path": "path/to/recipes" |
221 }""".lstrip() | 215 }""".lstrip() |
222 | 216 |
223 self.a = 'a'*40 | 217 self.a = 'a'*40 |
224 self.a_dat = { | 218 self.a_dat = { |
225 'commit': self.a, | 219 'commit': self.a, |
226 'author': {'email': 'foo@example.com'}, | 220 'author': {'email': 'foo@example.com'}, |
227 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, | 221 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, |
228 'message': 'message', | 222 'message': 'message', |
229 'tree_diff': [ | |
230 { | |
231 'old_path': 'unrelated', | |
232 'new_path': 'unrelated', | |
233 }, | |
234 { | |
235 'old_path': 'path/to/recipes/foo', | |
236 'new_path': 'path/to/recipes/bar', | |
237 }, | |
238 ] | |
239 } | 223 } |
240 | 224 |
241 self.a_meta = fetch.CommitMetadata( | 225 self.a_meta = fetch.CommitMetadata( |
242 revision = 'a'*40, | 226 revision = 'a'*40, |
243 author_email = 'foo@example.com', | 227 author_email = 'foo@example.com', |
244 commit_timestamp = 1492131405, | 228 commit_timestamp = 1492131405, |
245 message_lines = ('message',), | 229 message_lines = ('message',), |
246 spec = package_pb2.Package( | 230 spec = package_pb2.Package( |
247 api_version = 2, | 231 api_version = 2, |
248 project_id = 'foo', | 232 project_id = 'foo', |
249 recipes_path = 'path/to/recipes', | 233 recipes_path = 'path/to/recipes', |
250 ), | 234 ) |
251 roll_candidate = True, | |
252 ) | 235 ) |
253 | 236 |
254 def assertMultiDone(self, mocked_call): | 237 def assertMultiDone(self, mocked_call): |
255 with self.assertRaises(NoMoreExpectatedCalls): | 238 with self.assertRaises(NoMoreExpectatedCalls): |
256 mocked_call() | 239 mocked_call() |
257 | 240 |
258 def j(self, url, data, status_code=200): | 241 def j(self, url, data, status_code=200): |
259 """Mock a request.get to return json data.""" | 242 """Mock a request.get to return json data.""" |
260 return self.r(url, u')]}\'\n'+json.dumps(data), status_code) | 243 return self.r(url, u')]}\'\n'+json.dumps(data), status_code) |
261 | 244 |
(...skipping 20 matching lines...) Expand all Loading... |
282 status_code=status_code) | 265 status_code=status_code) |
283 return _inner | 266 return _inner |
284 | 267 |
285 @mock.patch('__builtin__.open', mock.mock_open()) | 268 @mock.patch('__builtin__.open', mock.mock_open()) |
286 @mock.patch('shutil.rmtree') | 269 @mock.patch('shutil.rmtree') |
287 @mock.patch('os.makedirs') | 270 @mock.patch('os.makedirs') |
288 @mock.patch('tarfile.open') | 271 @mock.patch('tarfile.open') |
289 @mock.patch('requests.get') | 272 @mock.patch('requests.get') |
290 def test_checkout(self, requests_get, _tarfile_open, makedirs, rmtree): | 273 def test_checkout(self, requests_get, _tarfile_open, makedirs, rmtree): |
291 requests_get.side_effect = multi( | 274 requests_get.side_effect = multi( |
292 self.j('repo/+/revision?name-status=1&format=JSON', self.a_dat), | 275 self.j('repo/+/revision?format=JSON', self.a_dat), |
293 self.j('repo/+/%s?name-status=1&format=JSON' % self.a, self.a_dat), | 276 self.j('repo/+/%s?format=JSON' % self.a, self.a_dat), |
294 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, | 277 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, |
295 self.proto_text), | 278 self.proto_text), |
296 self.d('repo/+archive/%s/path/to/recipes.tar.gz' % self.a, ''), | 279 self.d('repo/+archive/%s/path/to/recipes.tar.gz' % self.a, ''), |
297 ) | 280 ) |
298 | 281 |
299 fetch.GitilesBackend('dir', 'repo', True).checkout('revision') | 282 fetch.GitilesBackend('dir', 'repo', True).checkout('revision') |
300 | 283 |
301 makedirs.assert_has_calls([ | 284 makedirs.assert_has_calls([ |
302 mock.call('dir/infra/config'), | 285 mock.call('dir/infra/config'), |
303 mock.call('dir/path/to/recipes'), | 286 mock.call('dir/path/to/recipes'), |
304 ]) | 287 ]) |
305 | 288 |
306 rmtree.assert_called_once_with('dir', ignore_errors=True) | 289 rmtree.assert_called_once_with('dir', ignore_errors=True) |
307 self.assertMultiDone(requests_get) | 290 self.assertMultiDone(requests_get) |
308 | 291 |
309 @mock.patch('requests.get') | 292 @mock.patch('requests.get') |
310 def test_updates(self, requests_get): | 293 def test_updates(self, requests_get): |
311 sha_a = 'a'*40 | 294 sha_a = 'a'*40 |
312 sha_b = 'b'*40 | 295 sha_b = 'b'*40 |
313 log_json = { | 296 log_json = { |
314 'log': [ | 297 'log': [ |
315 { | 298 { |
316 'commit': sha_b, | 299 'commit': sha_b, |
317 'author': {'email': 'foo@example.com'}, | 300 'author': {'email': 'foo@example.com'}, |
318 'committer': {'time': 'Fri Apr 14 00:58:45 2017'}, | 301 'committer': {'time': 'Fri Apr 14 00:58:45 2017'}, |
319 'message': 'message', | 302 'message': 'message', |
320 'tree_diff': [ | 303 'tree_diff': [ |
321 { | 304 { |
322 'old_path': '/dev/null', | 305 'old_path': '/dev/null', |
323 'new_path': 'path/to/recipes/path1/foo', | 306 'new_path': 'path1/foo', |
324 }, | 307 }, |
325 ], | 308 ], |
326 }, | 309 }, |
327 { | 310 { |
328 'commit': 'def456', | 311 'commit': 'def456', |
329 'author': {'email': 'foo@example.com'}, | 312 'author': {'email': 'foo@example.com'}, |
330 'committer': {'time': 'Fri Apr 14 00:57:45 2017'}, | 313 'committer': {'time': 'Fri Apr 14 00:57:45 2017'}, |
331 'message': 'message', | 314 'message': 'message', |
332 'tree_diff': [ | 315 'tree_diff': [ |
333 { | 316 { |
334 'old_path': '/dev/null', | 317 'old_path': '/dev/null', |
335 'new_path': 'path8/foo', | 318 'new_path': 'path8/foo', |
336 }, | 319 }, |
337 ], | 320 ], |
338 }, | 321 }, |
339 { | 322 { |
340 'commit': sha_a, | 323 'commit': sha_a, |
341 'author': {'email': 'foo@example.com'}, | 324 'author': {'email': 'foo@example.com'}, |
342 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, | 325 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, |
343 'message': 'message', | 326 'message': 'message', |
344 'tree_diff': [ | 327 'tree_diff': [ |
345 { | 328 { |
346 'old_path': '/dev/null', | 329 'old_path': '/dev/null', |
347 'new_path': 'path/to/recipes/path8/foo', | 330 'new_path': 'path8/foo', |
348 }, | 331 }, |
349 { | 332 { |
350 'old_path': 'path2/foo', | 333 'old_path': 'path2/foo', |
351 'new_path': '/dev/null', | 334 'new_path': '/dev/null', |
352 }, | 335 }, |
353 ], | 336 ], |
354 }, | 337 }, |
355 ], | 338 ], |
356 } | 339 } |
357 | 340 |
358 requests_get.side_effect = multi( | 341 requests_get.side_effect = multi( |
359 self.j('repo/+/reva?name-status=1&format=JSON', log_json['log'][2]), | 342 self.j('repo/+/reva?format=JSON', { |
360 self.j('repo/+/revb?name-status=1&format=JSON', log_json['log'][0]), | 343 'commit': sha_a, |
| 344 'author': {'email': 'foo@example.com'}, |
| 345 'committer': {'time': 'Fri Apr 14 00:56:45 2017'}, |
| 346 'message': 'message', |
| 347 }), |
| 348 self.j('repo/+/revb?format=JSON', { |
| 349 'commit': sha_b, |
| 350 'author': {'email': 'foo@example.com'}, |
| 351 'committer': {'time': 'Fri Apr 14 00:58:45 2017'}, |
| 352 'message': 'message', |
| 353 }), |
361 self.j('repo/+log/%s..%s?name-status=1&format=JSON' % (sha_a, sha_b), | 354 self.j('repo/+log/%s..%s?name-status=1&format=JSON' % (sha_a, sha_b), |
362 log_json), | 355 log_json), |
363 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_a, | 356 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_a, |
364 self.proto_text), | 357 self.proto_text), |
365 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % 'def456', | |
366 self.proto_text), | |
367 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_b, | 358 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % sha_b, |
368 self.proto_text), | 359 self.proto_text), |
369 ) | 360 ) |
370 | 361 |
371 be = fetch.GitilesBackend('dir', 'repo', True) | 362 be = fetch.GitilesBackend('dir', 'repo', True) |
372 self.assertEqual(sha_a, be.resolve_refspec('reva')) | 363 self.assertEqual(sha_a, be.resolve_refspec('reva')) |
373 self.assertEqual(sha_b, be.resolve_refspec('revb')) | 364 self.assertEqual(sha_b, be.resolve_refspec('revb')) |
374 | 365 |
375 self.assertEqual( | 366 self.assertEqual( |
376 [self.a_meta, | 367 [self.a_meta, |
377 fetch.CommitMetadata( | 368 fetch.CommitMetadata( |
378 revision = 'def456', | |
379 author_email = 'foo@example.com', | |
380 commit_timestamp = 1492131465, | |
381 message_lines = ('message',), | |
382 spec = package_pb2.Package( | |
383 api_version = 2, | |
384 project_id = 'foo', | |
385 recipes_path = 'path/to/recipes', | |
386 ), | |
387 roll_candidate = False, | |
388 ), | |
389 fetch.CommitMetadata( | |
390 revision = sha_b, | 369 revision = sha_b, |
391 author_email = 'foo@example.com', | 370 author_email = 'foo@example.com', |
392 commit_timestamp = 1492131525, | 371 commit_timestamp = 1492131525, |
393 message_lines = ('message',), | 372 message_lines = ('message',), |
394 spec = package_pb2.Package( | 373 spec = package_pb2.Package( |
395 api_version = 2, | 374 api_version = 2, |
396 project_id = 'foo', | 375 project_id = 'foo', |
397 recipes_path = 'path/to/recipes', | 376 recipes_path = 'path/to/recipes', |
398 ), | 377 ) |
399 roll_candidate = True, | |
400 )], | 378 )], |
401 be.updates(sha_a, sha_b)) | 379 be.updates(sha_a, sha_b, ['path1', 'path2'])) |
402 self.assertMultiDone(requests_get) | 380 self.assertMultiDone(requests_get) |
403 | 381 |
404 | 382 |
405 @mock.patch('requests.get') | 383 @mock.patch('requests.get') |
406 def test_commit_metadata(self, requests_get): | 384 def test_commit_metadata(self, requests_get): |
407 requests_get.side_effect = multi( | 385 requests_get.side_effect = multi( |
408 self.j('repo/+/revision?name-status=1&format=JSON', self.a_dat), | 386 self.j('repo/+/revision?format=JSON', self.a_dat), |
409 self.j('repo/+/%s?name-status=1&format=JSON' % self.a, self.a_dat), | 387 self.j('repo/+/%s?format=JSON' % self.a, self.a_dat), |
410 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, | 388 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, |
411 self.proto_text) | 389 self.proto_text) |
412 ) | 390 ) |
413 | 391 |
414 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( | 392 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( |
415 'revision') | 393 'revision') |
416 self.assertEqual(result, self.a_meta) | 394 self.assertEqual(result, self.a_meta) |
417 self.assertMultiDone(requests_get) | 395 self.assertMultiDone(requests_get) |
418 | 396 |
419 @mock.patch('requests.get') | 397 @mock.patch('requests.get') |
420 def test_non_transient_error(self, requests_get): | 398 def test_non_transient_error(self, requests_get): |
421 requests_get.side_effect = multi( | 399 requests_get.side_effect = multi( |
422 self.r('repo/+/revision?name-status=1&format=JSON', | 400 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(403, '')), |
423 fetch.GitilesFetchError(403, '')), | |
424 ) | 401 ) |
425 with self.assertRaises(fetch.GitilesFetchError): | 402 with self.assertRaises(fetch.GitilesFetchError): |
426 fetch.GitilesBackend('dir', 'repo', True).commit_metadata( | 403 fetch.GitilesBackend('dir', 'repo', True).commit_metadata( |
427 'revision') | 404 'revision') |
428 self.assertMultiDone(requests_get) | 405 self.assertMultiDone(requests_get) |
429 | 406 |
430 @mock.patch('requests.get') | 407 @mock.patch('requests.get') |
431 @mock.patch('time.sleep') | 408 @mock.patch('time.sleep') |
432 @mock.patch('logging.exception') | 409 @mock.patch('logging.exception') |
433 def test_transient_retry(self, _logging_exception, _time_sleep, requests_get): | 410 def test_transient_retry(self, _logging_exception, _time_sleep, requests_get): |
434 requests_get.side_effect = multi( | 411 requests_get.side_effect = multi( |
435 self.r('repo/+/revision?name-status=1&format=JSON', | 412 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), |
436 fetch.GitilesFetchError(500, '')), | 413 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), |
437 self.r('repo/+/revision?name-status=1&format=JSON', | 414 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), |
438 fetch.GitilesFetchError(500, '')), | 415 self.r('repo/+/revision?format=JSON', fetch.GitilesFetchError(500, '')), |
439 self.r('repo/+/revision?name-status=1&format=JSON', | 416 self.j('repo/+/revision?format=JSON', self.a_dat), |
440 fetch.GitilesFetchError(500, '')), | 417 self.j('repo/+/%s?format=JSON' % self.a, self.a_dat), |
441 self.r('repo/+/revision?name-status=1&format=JSON', | |
442 fetch.GitilesFetchError(500, '')), | |
443 self.j('repo/+/revision?name-status=1&format=JSON', self.a_dat), | |
444 self.j('repo/+/%s?name-status=1&format=JSON' % self.a, self.a_dat), | |
445 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, | 418 self.d('repo/+/%s/infra/config/recipes.cfg?format=TEXT' % self.a, |
446 self.proto_text), | 419 self.proto_text), |
447 ) | 420 ) |
448 | 421 |
449 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( | 422 result = fetch.GitilesBackend('dir', 'repo', True).commit_metadata( |
450 'revision') | 423 'revision') |
451 self.assertEqual(result, self.a_meta) | 424 self.assertEqual(result, self.a_meta) |
452 self.assertMultiDone(requests_get) | 425 self.assertMultiDone(requests_get) |
453 | 426 |
454 if __name__ == '__main__': | 427 if __name__ == '__main__': |
455 unittest.main() | 428 unittest.main() |
OLD | NEW |