| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2011 The Native Client Authors. All rights reserved. | 2 # Copyright (c) 2011 The Native Client Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Script to synchronise the naclports mirror of upstream archives. | 6 """Script to synchronise the naclports mirror of upstream archives. |
| 7 | 7 |
| 8 This script verifies that the URL for every package is mirrored on | 8 This script verifies that the URL for every package is mirrored on |
| 9 Google Cloud Storage. If it finds missing URLs it downloads them to | 9 Google Cloud Storage. If it finds missing URLs it downloads them to |
| 10 the local machine and then pushes them up using gsutil. | 10 the local machine and then pushes them up using gsutil. |
| (...skipping 15 matching lines...) Expand all Loading... |
| 26 | 26 |
| 27 import naclports | 27 import naclports |
| 28 import naclports.source_package | 28 import naclports.source_package |
| 29 | 29 |
| 30 MIRROR_GS = 'gs://naclports/mirror' | 30 MIRROR_GS = 'gs://naclports/mirror' |
| 31 | 31 |
| 32 | 32 |
| 33 def GsUpload(options, filename, url): | 33 def GsUpload(options, filename, url): |
| 34 """Upload a file to Google cloud storage using gsutil""" | 34 """Upload a file to Google cloud storage using gsutil""" |
| 35 naclports.Log("Uploading to mirror: %s" % url) | 35 naclports.Log("Uploading to mirror: %s" % url) |
| 36 cmd = ['gsutil', 'cp', '-a', 'public-read', filename, url] | 36 cmd = [options.gsutil, 'cp', '-a', 'public-read', filename, url] |
| 37 if options.dry_run: | 37 if options.dry_run: |
| 38 naclports.Log(cmd) | 38 naclports.Log(cmd) |
| 39 else: | 39 else: |
| 40 subprocess.check_call(cmd) | 40 subprocess.check_call(cmd) |
| 41 | 41 |
| 42 | 42 |
| 43 def GetMirrorListing(url): | 43 def GetMirrorListing(options, url): |
| 44 """Get filename listing for a Google cloud storage URL""" | 44 """Get filename listing for a Google cloud storage URL""" |
| 45 listing = subprocess.check_output(['gsutil', 'ls', url]) | 45 listing = subprocess.check_output([options.gsutil, 'ls', url]) |
| 46 listing = listing.splitlines() | 46 listing = listing.splitlines() |
| 47 listing = [os.path.basename(l) for l in listing] | 47 listing = [os.path.basename(l) for l in listing] |
| 48 return listing | 48 return listing |
| 49 | 49 |
| 50 | 50 |
| 51 def CheckMirror(options, package, mirror_listing): | 51 def CheckMirror(options, package, mirror_listing): |
| 52 """Check that is package has is archive mirrors on Google cloud storage""" | 52 """Check that is package has is archive mirrors on Google cloud storage""" |
| 53 naclports.Trace('Checking %s' % package.NAME) | 53 naclports.Trace('Checking %s' % package.NAME) |
| 54 basename = package.GetArchiveFilename() | 54 basename = package.GetArchiveFilename() |
| 55 if not basename: | 55 if not basename: |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 87 parser.add_argument('-n', '--dry-run', action='store_true', | 87 parser.add_argument('-n', '--dry-run', action='store_true', |
| 88 help="Don't actually upload anything") | 88 help="Don't actually upload anything") |
| 89 parser.add_argument('--check', action='store_true', | 89 parser.add_argument('--check', action='store_true', |
| 90 help='Verify that the mirror is up-to-date.') | 90 help='Verify that the mirror is up-to-date.') |
| 91 parser.add_argument('-v', '--verbose', action='store_true', | 91 parser.add_argument('-v', '--verbose', action='store_true', |
| 92 help='Enable verbose output.') | 92 help='Enable verbose output.') |
| 93 options = parser.parse_args(args) | 93 options = parser.parse_args(args) |
| 94 naclports.SetVerbose(options.verbose) | 94 naclports.SetVerbose(options.verbose) |
| 95 | 95 |
| 96 # Ensure gsutil is in the PATH. | 96 # Ensure gsutil is in the PATH. |
| 97 naclports.util.FindInPath('gsutil') | 97 bot_gsutil = '/b/build/scripts/slave/gsutil' |
| 98 if os.path.exists(bot_gsutil): |
| 99 gsutil = bot_gsutil |
| 100 else: |
| 101 gsutil = 'gsutil' |
| 102 naclports.util.FindInPath(gsutil) |
| 103 options.gsutil = gsutil |
| 98 | 104 |
| 99 listing = GetMirrorListing(MIRROR_GS) | 105 listing = GetMirrorListing(options, MIRROR_GS) |
| 100 source_packages = naclports.source_package.SourcePackageIterator() | 106 source_packages = naclports.source_package.SourcePackageIterator() |
| 101 | 107 |
| 102 return CheckPackages(options, source_packages, listing) | 108 return CheckPackages(options, source_packages, listing) |
| 103 | 109 |
| 104 | 110 |
| 105 if __name__ == '__main__': | 111 if __name__ == '__main__': |
| 106 try: | 112 try: |
| 107 sys.exit(main(sys.argv[1:])) | 113 sys.exit(main(sys.argv[1:])) |
| 108 except naclports.Error as e: | 114 except naclports.Error as e: |
| 109 sys.stderr.write('%s\n' % e) | 115 sys.stderr.write('%s\n' % e) |
| 110 sys.exit(-1) | 116 sys.exit(-1) |
| OLD | NEW |