| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2014 The Native Client Authors. All rights reserved. | 2 # Copyright (c) 2014 The Native Client Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Scan online binary packages to produce new package index. | 6 """Scan online binary packages to produce new package index. |
| 7 | 7 |
| 8 This script is indended to be run periodically and the | 8 This script is indended to be run periodically and the |
| 9 results checked into source control. This script depends on | 9 results checked into source control. This script depends on |
| 10 gsutil being installed. | 10 gsutil being installed. |
| 11 """ | 11 """ |
| 12 | 12 |
| 13 from __future__ import print_function | 13 from __future__ import print_function |
| 14 | 14 |
| 15 import argparse | 15 import argparse |
| 16 import collections | 16 import collections |
| 17 import hashlib | 17 import hashlib |
| 18 import os | 18 import os |
| 19 import subprocess | 19 import subprocess |
| 20 import sys | 20 import sys |
| 21 | 21 |
| 22 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | 22 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 23 sys.path.append(os.path.join(os.path.dirname(SCRIPT_DIR), 'lib')) | 23 sys.path.append(os.path.join(os.path.dirname(SCRIPT_DIR), 'lib')) |
| 24 | 24 |
| 25 import naclports | 25 import naclports |
| 26 import naclports.package | 26 import naclports.package |
| 27 import naclports.package_index | 27 import naclports.package_index |
| 28 | 28 |
| 29 from naclports import Log, Trace | 29 from naclports.util import Log, LogVerbose |
| 30 | 30 |
| 31 | 31 |
| 32 def FormatSize(num_bytes): | 32 def FormatSize(num_bytes): |
| 33 """Create a human readable string from a byte count.""" | 33 """Create a human readable string from a byte count.""" |
| 34 for x in ('bytes', 'KB', 'MB', 'GB', 'TB'): | 34 for x in ('bytes', 'KB', 'MB', 'GB', 'TB'): |
| 35 if num_bytes < 1024.0: | 35 if num_bytes < 1024.0: |
| 36 return "%3.1f %s" % (num_bytes, x) | 36 return "%3.1f %s" % (num_bytes, x) |
| 37 num_bytes /= 1024.0 | 37 num_bytes /= 1024.0 |
| 38 | 38 |
| 39 | 39 |
| (...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 134 args.revision) | 134 args.revision) |
| 135 gs_url = 'gs://' + base_path | 135 gs_url = 'gs://' + base_path |
| 136 listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt') | 136 listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt') |
| 137 if args.cache_listing and os.path.exists(listing_file): | 137 if args.cache_listing and os.path.exists(listing_file): |
| 138 Log('Using pre-cached gs listing: %s' % listing_file) | 138 Log('Using pre-cached gs listing: %s' % listing_file) |
| 139 with open(listing_file) as f: | 139 with open(listing_file) as f: |
| 140 listing = f.read() | 140 listing = f.read() |
| 141 else: | 141 else: |
| 142 Log("Searching for packages at: %s" % gs_url) | 142 Log("Searching for packages at: %s" % gs_url) |
| 143 cmd = ['gsutil', 'ls', '-le', gs_url] | 143 cmd = ['gsutil', 'ls', '-le', gs_url] |
| 144 Trace("Running: %s" % str(cmd)) | 144 LogVerbose("Running: %s" % str(cmd)) |
| 145 try: | 145 try: |
| 146 listing = subprocess.check_output(cmd) | 146 listing = subprocess.check_output(cmd) |
| 147 except subprocess.CalledProcessError as e: | 147 except subprocess.CalledProcessError as e: |
| 148 naclports.Error(e) | 148 naclports.Error(e) |
| 149 return 1 | 149 return 1 |
| 150 | 150 |
| 151 all_files = ParseGsUtilLs(listing) | 151 all_files = ParseGsUtilLs(listing) |
| 152 if args.cache_listing and not os.path.exists(listing_file): | 152 if args.cache_listing and not os.path.exists(listing_file): |
| 153 with open(listing_file, 'w') as f: | 153 with open(listing_file, 'w') as f: |
| 154 f.write(listing) | 154 f.write(listing) |
| 155 | 155 |
| 156 Log('Found %d packages [%s]' % (len(all_files), | 156 Log('Found %d packages [%s]' % (len(all_files), |
| 157 FormatSize(sum(f.size for f in all_files)))) | 157 FormatSize(sum(f.size for f in all_files)))) |
| 158 | 158 |
| 159 binaries = DownloadFiles(all_files, not args.skip_md5) | 159 binaries = DownloadFiles(all_files, not args.skip_md5) |
| 160 index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt') | 160 index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt') |
| 161 Log('Generating %s' % index_file) | 161 Log('Generating %s' % index_file) |
| 162 naclports.package_index.WriteIndex(index_file, binaries) | 162 naclports.package_index.WriteIndex(index_file, binaries) |
| 163 Log('Done') | 163 Log('Done') |
| 164 return 0 | 164 return 0 |
| 165 | 165 |
| 166 | 166 |
| 167 if __name__ == '__main__': | 167 if __name__ == '__main__': |
| 168 sys.exit(main(sys.argv[1:])) | 168 sys.exit(main(sys.argv[1:])) |
| OLD | NEW |