Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(26)

Side by Side Diff: build_tools/scan_packages.py

Issue 1012773005: Improve tracing/logging system used in lib/naclports (Closed) Base URL: https://chromium.googlesource.com/external/naclports.git@master
Patch Set: Created 5 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2014 The Native Client Authors. All rights reserved. 2 # Copyright (c) 2014 The Native Client Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Scan online binary packages to produce new package index. 6 """Scan online binary packages to produce new package index.
7 7
8 This script is indended to be run periodically and the 8 This script is indended to be run periodically and the
9 results checked into source control. This script depends on 9 results checked into source control. This script depends on
10 gsutil being installed. 10 gsutil being installed.
11 """ 11 """
12 12
13 from __future__ import print_function 13 from __future__ import print_function
14 14
15 import argparse 15 import argparse
16 import collections 16 import collections
17 import hashlib 17 import hashlib
18 import os 18 import os
19 import subprocess 19 import subprocess
20 import sys 20 import sys
21 21
22 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 22 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
23 sys.path.append(os.path.join(os.path.dirname(SCRIPT_DIR), 'lib')) 23 sys.path.append(os.path.join(os.path.dirname(SCRIPT_DIR), 'lib'))
24 24
25 import naclports 25 import naclports
26 import naclports.package 26 import naclports.package
27 import naclports.package_index 27 import naclports.package_index
28 28
29 from naclports import Log, Trace 29 from naclports.util import Log, LogVerbose
30 30
31 31
32 def FormatSize(num_bytes): 32 def FormatSize(num_bytes):
33 """Create a human readable string from a byte count.""" 33 """Create a human readable string from a byte count."""
34 for x in ('bytes', 'KB', 'MB', 'GB', 'TB'): 34 for x in ('bytes', 'KB', 'MB', 'GB', 'TB'):
35 if num_bytes < 1024.0: 35 if num_bytes < 1024.0:
36 return "%3.1f %s" % (num_bytes, x) 36 return "%3.1f %s" % (num_bytes, x)
37 num_bytes /= 1024.0 37 num_bytes /= 1024.0
38 38
39 39
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after
134 args.revision) 134 args.revision)
135 gs_url = 'gs://' + base_path 135 gs_url = 'gs://' + base_path
136 listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt') 136 listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt')
137 if args.cache_listing and os.path.exists(listing_file): 137 if args.cache_listing and os.path.exists(listing_file):
138 Log('Using pre-cached gs listing: %s' % listing_file) 138 Log('Using pre-cached gs listing: %s' % listing_file)
139 with open(listing_file) as f: 139 with open(listing_file) as f:
140 listing = f.read() 140 listing = f.read()
141 else: 141 else:
142 Log("Searching for packages at: %s" % gs_url) 142 Log("Searching for packages at: %s" % gs_url)
143 cmd = ['gsutil', 'ls', '-le', gs_url] 143 cmd = ['gsutil', 'ls', '-le', gs_url]
144 Trace("Running: %s" % str(cmd)) 144 LogVerbose("Running: %s" % str(cmd))
145 try: 145 try:
146 listing = subprocess.check_output(cmd) 146 listing = subprocess.check_output(cmd)
147 except subprocess.CalledProcessError as e: 147 except subprocess.CalledProcessError as e:
148 naclports.Error(e) 148 naclports.Error(e)
149 return 1 149 return 1
150 150
151 all_files = ParseGsUtilLs(listing) 151 all_files = ParseGsUtilLs(listing)
152 if args.cache_listing and not os.path.exists(listing_file): 152 if args.cache_listing and not os.path.exists(listing_file):
153 with open(listing_file, 'w') as f: 153 with open(listing_file, 'w') as f:
154 f.write(listing) 154 f.write(listing)
155 155
156 Log('Found %d packages [%s]' % (len(all_files), 156 Log('Found %d packages [%s]' % (len(all_files),
157 FormatSize(sum(f.size for f in all_files)))) 157 FormatSize(sum(f.size for f in all_files))))
158 158
159 binaries = DownloadFiles(all_files, not args.skip_md5) 159 binaries = DownloadFiles(all_files, not args.skip_md5)
160 index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt') 160 index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt')
161 Log('Generating %s' % index_file) 161 Log('Generating %s' % index_file)
162 naclports.package_index.WriteIndex(index_file, binaries) 162 naclports.package_index.WriteIndex(index_file, binaries)
163 Log('Done') 163 Log('Done')
164 return 0 164 return 0
165 165
166 166
167 if __name__ == '__main__': 167 if __name__ == '__main__':
168 sys.exit(main(sys.argv[1:])) 168 sys.exit(main(sys.argv[1:]))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698