OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2011 The Native Client Authors. All rights reserved. | 2 # Copyright (c) 2011 The Native Client Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Script to synchronise the naclports mirror of upstream archives. | 6 """Script to synchronise the naclports mirror of upstream archives. |
7 | 7 |
8 This script verifies that the URL for every package is mirrored on | 8 This script verifies that the URL for every package is mirrored on |
9 Google Cloud Storage. If it finds missing URLs it downloads them to | 9 Google Cloud Storage. If it finds missing URLs it downloads them to |
10 the local machine and then pushes them up using gsutil. | 10 the local machine and then pushes them up using gsutil. |
(...skipping 15 matching lines...) Expand all Loading... |
26 | 26 |
27 import naclports | 27 import naclports |
28 import naclports.source_package | 28 import naclports.source_package |
29 | 29 |
30 MIRROR_GS = 'gs://naclports/mirror' | 30 MIRROR_GS = 'gs://naclports/mirror' |
31 | 31 |
32 | 32 |
33 def GsUpload(options, filename, url): | 33 def GsUpload(options, filename, url): |
34 """Upload a file to Google cloud storage using gsutil""" | 34 """Upload a file to Google cloud storage using gsutil""" |
35 naclports.Log("Uploading to mirror: %s" % url) | 35 naclports.Log("Uploading to mirror: %s" % url) |
36 cmd = ['gsutil', 'cp', '-a', 'public-read', filename, url] | 36 cmd = [options.gsutil, 'cp', '-a', 'public-read', filename, url] |
37 if options.dry_run: | 37 if options.dry_run: |
38 naclports.Log(cmd) | 38 naclports.Log(cmd) |
39 else: | 39 else: |
40 subprocess.check_call(cmd) | 40 subprocess.check_call(cmd) |
41 | 41 |
42 | 42 |
43 def GetMirrorListing(url): | 43 def GetMirrorListing(options, url): |
44 """Get filename listing for a Google cloud storage URL""" | 44 """Get filename listing for a Google cloud storage URL""" |
45 listing = subprocess.check_output(['gsutil', 'ls', url]) | 45 listing = subprocess.check_output([options.gsutil, 'ls', url]) |
46 listing = listing.splitlines() | 46 listing = listing.splitlines() |
47 listing = [os.path.basename(l) for l in listing] | 47 listing = [os.path.basename(l) for l in listing] |
48 return listing | 48 return listing |
49 | 49 |
50 | 50 |
51 def CheckMirror(options, package, mirror_listing): | 51 def CheckMirror(options, package, mirror_listing): |
52 """Check that is package has is archive mirrors on Google cloud storage""" | 52 """Check that is package has is archive mirrors on Google cloud storage""" |
53 naclports.Trace('Checking %s' % package.NAME) | 53 naclports.Trace('Checking %s' % package.NAME) |
54 basename = package.GetArchiveFilename() | 54 basename = package.GetArchiveFilename() |
55 if not basename: | 55 if not basename: |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
87 parser.add_argument('-n', '--dry-run', action='store_true', | 87 parser.add_argument('-n', '--dry-run', action='store_true', |
88 help="Don't actually upload anything") | 88 help="Don't actually upload anything") |
89 parser.add_argument('--check', action='store_true', | 89 parser.add_argument('--check', action='store_true', |
90 help='Verify that the mirror is up-to-date.') | 90 help='Verify that the mirror is up-to-date.') |
91 parser.add_argument('-v', '--verbose', action='store_true', | 91 parser.add_argument('-v', '--verbose', action='store_true', |
92 help='Enable verbose output.') | 92 help='Enable verbose output.') |
93 options = parser.parse_args(args) | 93 options = parser.parse_args(args) |
94 naclports.SetVerbose(options.verbose) | 94 naclports.SetVerbose(options.verbose) |
95 | 95 |
96 # Ensure gsutil is in the PATH. | 96 # Ensure gsutil is in the PATH. |
97 naclports.util.FindInPath('gsutil') | 97 bot_gsutil = '/b/build/scripts/slave/gsutil' |
| 98 if os.path.exists(bot_gsutil): |
| 99 gsutil = bot_gsutil |
| 100 else: |
| 101 gsutil = 'gsutil' |
| 102 naclports.util.FindInPath(gsutil) |
| 103 options.gsutil = gsutil |
98 | 104 |
99 listing = GetMirrorListing(MIRROR_GS) | 105 listing = GetMirrorListing(options, MIRROR_GS) |
100 source_packages = naclports.source_package.SourcePackageIterator() | 106 source_packages = naclports.source_package.SourcePackageIterator() |
101 | 107 |
102 return CheckPackages(options, source_packages, listing) | 108 return CheckPackages(options, source_packages, listing) |
103 | 109 |
104 | 110 |
105 if __name__ == '__main__': | 111 if __name__ == '__main__': |
106 try: | 112 try: |
107 sys.exit(main(sys.argv[1:])) | 113 sys.exit(main(sys.argv[1:])) |
108 except naclports.Error as e: | 114 except naclports.Error as e: |
109 sys.stderr.write('%s\n' % e) | 115 sys.stderr.write('%s\n' % e) |
110 sys.exit(-1) | 116 sys.exit(-1) |
OLD | NEW |