OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2014 The Native Client Authors. All rights reserved. | 2 # Copyright (c) 2014 The Native Client Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Scan online binary packages to produce new package index. | 6 """Scan online binary packages to produce new package index. |
7 | 7 |
8 This script is indended to be run periodically and the | 8 This script is indended to be run periodically and the |
9 results checked into source control. This script depends on | 9 results checked into source control. This script depends on |
10 gsutil being installed. | 10 gsutil being installed. |
11 """ | 11 """ |
12 | 12 |
13 from __future__ import print_function | 13 from __future__ import print_function |
14 | 14 |
15 import argparse | 15 import argparse |
16 import collections | 16 import collections |
17 import hashlib | 17 import hashlib |
18 import os | 18 import os |
19 import subprocess | 19 import subprocess |
20 import sys | 20 import sys |
21 | 21 |
22 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | 22 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
23 sys.path.append(os.path.join(os.path.dirname(SCRIPT_DIR), 'lib')) | 23 sys.path.append(os.path.join(os.path.dirname(SCRIPT_DIR), 'lib')) |
24 | 24 |
25 import naclports | 25 import naclports |
26 import naclports.package | 26 import naclports.package |
27 import naclports.package_index | 27 import naclports.package_index |
28 | 28 |
29 from naclports import Log, Trace | 29 from naclports.util import Log, LogVerbose |
30 | 30 |
31 | 31 |
32 def FormatSize(num_bytes): | 32 def FormatSize(num_bytes): |
33 """Create a human readable string from a byte count.""" | 33 """Create a human readable string from a byte count.""" |
34 for x in ('bytes', 'KB', 'MB', 'GB', 'TB'): | 34 for x in ('bytes', 'KB', 'MB', 'GB', 'TB'): |
35 if num_bytes < 1024.0: | 35 if num_bytes < 1024.0: |
36 return "%3.1f %s" % (num_bytes, x) | 36 return "%3.1f %s" % (num_bytes, x) |
37 num_bytes /= 1024.0 | 37 num_bytes /= 1024.0 |
38 | 38 |
39 | 39 |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
134 args.revision) | 134 args.revision) |
135 gs_url = 'gs://' + base_path | 135 gs_url = 'gs://' + base_path |
136 listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt') | 136 listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt') |
137 if args.cache_listing and os.path.exists(listing_file): | 137 if args.cache_listing and os.path.exists(listing_file): |
138 Log('Using pre-cached gs listing: %s' % listing_file) | 138 Log('Using pre-cached gs listing: %s' % listing_file) |
139 with open(listing_file) as f: | 139 with open(listing_file) as f: |
140 listing = f.read() | 140 listing = f.read() |
141 else: | 141 else: |
142 Log("Searching for packages at: %s" % gs_url) | 142 Log("Searching for packages at: %s" % gs_url) |
143 cmd = ['gsutil', 'ls', '-le', gs_url] | 143 cmd = ['gsutil', 'ls', '-le', gs_url] |
144 Trace("Running: %s" % str(cmd)) | 144 LogVerbose("Running: %s" % str(cmd)) |
145 try: | 145 try: |
146 listing = subprocess.check_output(cmd) | 146 listing = subprocess.check_output(cmd) |
147 except subprocess.CalledProcessError as e: | 147 except subprocess.CalledProcessError as e: |
148 naclports.Error(e) | 148 naclports.Error(e) |
149 return 1 | 149 return 1 |
150 | 150 |
151 all_files = ParseGsUtilLs(listing) | 151 all_files = ParseGsUtilLs(listing) |
152 if args.cache_listing and not os.path.exists(listing_file): | 152 if args.cache_listing and not os.path.exists(listing_file): |
153 with open(listing_file, 'w') as f: | 153 with open(listing_file, 'w') as f: |
154 f.write(listing) | 154 f.write(listing) |
155 | 155 |
156 Log('Found %d packages [%s]' % (len(all_files), | 156 Log('Found %d packages [%s]' % (len(all_files), |
157 FormatSize(sum(f.size for f in all_files)))) | 157 FormatSize(sum(f.size for f in all_files)))) |
158 | 158 |
159 binaries = DownloadFiles(all_files, not args.skip_md5) | 159 binaries = DownloadFiles(all_files, not args.skip_md5) |
160 index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt') | 160 index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt') |
161 Log('Generating %s' % index_file) | 161 Log('Generating %s' % index_file) |
162 naclports.package_index.WriteIndex(index_file, binaries) | 162 naclports.package_index.WriteIndex(index_file, binaries) |
163 Log('Done') | 163 Log('Done') |
164 return 0 | 164 return 0 |
165 | 165 |
166 | 166 |
167 if __name__ == '__main__': | 167 if __name__ == '__main__': |
168 sys.exit(main(sys.argv[1:])) | 168 sys.exit(main(sys.argv[1:])) |
OLD | NEW |