| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/python | |
| 2 # Copyright (c) 2012 The Native Client Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 import hashlib | |
| 7 import optparse | |
| 8 import os | |
| 9 import shutil | |
| 10 import struct | |
| 11 import subprocess | |
| 12 import sys | |
| 13 import tempfile | |
| 14 import time | |
| 15 | |
| 16 | |
| 17 KNOWN_BAD_NEXES = set(['naclapps/nexes/' + i for i in [ | |
| 18 ('kgegkcnohlcckapfehpogebcncidcdbe/' | |
| 19 '1c2b051cb60367cf103128c9cd76769ffa1cf356.nexe'), | |
| 20 ('noiedkpkpicokakliamgmmbmhkmkcdgj/' | |
| 21 '1c2b051cb60367cf103128c9cd76769ffa1cf356.nexe'), | |
| 22 ]]) | |
| 23 | |
| 24 | |
| 25 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| 26 TESTS_DIR = os.path.dirname(SCRIPT_DIR) | |
| 27 NACL_DIR = os.path.dirname(TESTS_DIR) | |
| 28 | |
| 29 | |
| 30 # Imports from the build directory. | |
| 31 sys.path.insert(0, os.path.join(NACL_DIR, 'build')) | |
| 32 import download_utils | |
| 33 | |
| 34 | |
| 35 def NexeArchitecture(filename): | |
| 36 """Decide the architecture of a nexe. | |
| 37 | |
| 38 Args: | |
| 39 filename: filename of the nexe. | |
| 40 Returns: | |
| 41 Architecture string (x86-32 / x86-64) or None. | |
| 42 """ | |
| 43 fh = open(filename, 'rb') | |
| 44 head = fh.read(20) | |
| 45 # Must not be too short. | |
| 46 if len(head) != 20: | |
| 47 print 'ERROR - header too short' | |
| 48 return None | |
| 49 # Must have ELF header. | |
| 50 if head[0:4] != '\x7fELF': | |
| 51 print 'ERROR - no elf header' | |
| 52 return None | |
| 53 # Decode e_machine | |
| 54 machine = struct.unpack('<H', head[18:])[0] | |
| 55 return { | |
| 56 3: 'x86-32', | |
| 57 #40: 'arm', # TODO(bradnelson): handle arm. | |
| 58 62: 'x86-64', | |
| 59 }.get(machine) | |
| 60 | |
| 61 | |
| 62 def GsutilCopySilent(src, dst): | |
| 63 """Invoke gsutil cp, swallowing the output, with retry. | |
| 64 | |
| 65 Args: | |
| 66 src: src url. | |
| 67 dst: dst path. | |
| 68 """ | |
| 69 for _ in range(3): | |
| 70 env = os.environ.copy() | |
| 71 env['PATH'] = '/b/build/scripts/slave' + os.pathsep + env['PATH'] | |
| 72 process = subprocess.Popen( | |
| 73 ['gsutil', 'cp', src, dst], | |
| 74 env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| 75 process_stdout, process_stderr = process.communicate() | |
| 76 if process.returncode == 0: | |
| 77 return | |
| 78 print 'Unexpected return code: %s' % process.returncode | |
| 79 print '>>> STDOUT' | |
| 80 print process_stdout | |
| 81 print '>>> STDERR' | |
| 82 print process_stderr | |
| 83 print '-' * 70 | |
| 84 sys.exit(1) | |
| 85 | |
| 86 | |
| 87 def DownloadNexeList(filename): | |
| 88 """Download list of NEXEs. | |
| 89 | |
| 90 Args: | |
| 91 filename: destination filename. | |
| 92 """ | |
| 93 GsutilCopySilent('gs://nativeclient-snaps/naclapps.list', filename) | |
| 94 | |
| 95 | |
| 96 def DownloadNexe(src_path, dst_filename): | |
| 97 """Download list of NEXEs. | |
| 98 | |
| 99 Args: | |
| 100 src_path: datastore relative path to download from. | |
| 101 dst_filename: destination filename. | |
| 102 """ | |
| 103 GsutilCopySilent('gs://nativeclient-snaps/%s' % src_path, dst_filename) | |
| 104 | |
| 105 | |
| 106 def Sha1Sum(path): | |
| 107 """Determine the sha1 hash of a file's contents given its path.""" | |
| 108 m = hashlib.sha1() | |
| 109 fh = open(path, 'rb') | |
| 110 m.update(fh.read()) | |
| 111 fh.close() | |
| 112 return m.hexdigest() | |
| 113 | |
| 114 | |
| 115 def ValidateNexe(options, path, src_path, expect_pass, print_errors): | |
| 116 """Run the validator on a nexe, check if the result is expected. | |
| 117 | |
| 118 Args: | |
| 119 options: bag of options. | |
| 120 path: path to the nexe. | |
| 121 src_path: path to nexe on server. | |
| 122 expect_pass: boolean indicating if the nexe is expected to validate. | |
| 123 print_errors: boolean indicating if detailed failure should be emitted. | |
| 124 Returns: | |
| 125 True if validation matches expectations. | |
| 126 """ | |
| 127 process = subprocess.Popen( | |
| 128 [options.validator, path], | |
| 129 stdout=subprocess.PIPE, | |
| 130 stderr=subprocess.PIPE) | |
| 131 process_stdout, process_stderr = process.communicate() | |
| 132 # Check if result is what we expect. | |
| 133 did_pass = (process.returncode == 0) | |
| 134 if expect_pass != did_pass: | |
| 135 if print_errors: | |
| 136 print '-' * 70 | |
| 137 print 'Validating: %s' % path | |
| 138 print 'From: %s' % src_path | |
| 139 print 'Size: %d' % os.path.getsize(path) | |
| 140 print 'SHA1: %s' % Sha1Sum(path) | |
| 141 print 'Validator: %s' % options.validator | |
| 142 print 'Unexpected return code: %s' % process.returncode | |
| 143 print '>>> STDOUT' | |
| 144 print process_stdout | |
| 145 print '>>> STDERR' | |
| 146 print process_stderr | |
| 147 print '-' * 70 | |
| 148 return False | |
| 149 return True | |
| 150 | |
| 151 | |
| 152 def NexeShouldValidate(path): | |
| 153 """Checks a blacklist to decide if a nexe should validate. | |
| 154 | |
| 155 Args: | |
| 156 path: path to the nexe. | |
| 157 Returns: | |
| 158 Boolean indicating if the nexe should validate. | |
| 159 """ | |
| 160 return path not in KNOWN_BAD_NEXES | |
| 161 | |
| 162 | |
| 163 def CachedPath(options, filename): | |
| 164 """Find the full path of a cached file, a cache root relative path. | |
| 165 | |
| 166 Args: | |
| 167 options: bags of options. | |
| 168 filename: filename relative to the top of the download url / cache. | |
| 169 Returns: | |
| 170 Absolute path of where the file goes in the cache. | |
| 171 """ | |
| 172 return os.path.join(options.cache_dir, 'nacl_validator_test_cache', filename) | |
| 173 | |
| 174 | |
| 175 def Sha1FromFilename(filename): | |
| 176 """Get the expected sha1 of a file path. | |
| 177 | |
| 178 Throughout we use the convention that files are store to a name of the form: | |
| 179 <path_to_file>/<sha1hex>[.<some_extention>] | |
| 180 This function extracts the expected sha1. | |
| 181 | |
| 182 Args: | |
| 183 filename: filename to extract. | |
| 184 Returns: | |
| 185 Excepted sha1. | |
| 186 """ | |
| 187 return os.path.splitext(os.path.basename(filename))[0] | |
| 188 | |
| 189 | |
| 190 def PrimeCache(options, filename): | |
| 191 """Attempt to add a file to the cache directory if its not already there. | |
| 192 | |
| 193 Args: | |
| 194 options: bag of options. | |
| 195 filename: filename relative to the top of the download url / cache. | |
| 196 """ | |
| 197 dpath = CachedPath(options, filename) | |
| 198 if not os.path.exists(dpath) or Sha1Sum(dpath) != Sha1FromFilename(filename): | |
| 199 # Try to make the directory, fail is ok, let the download fail instead. | |
| 200 try: | |
| 201 os.makedirs(os.path.basename(dpath)) | |
| 202 except OSError: | |
| 203 pass | |
| 204 DownloadNexe(filename, dpath) | |
| 205 | |
| 206 | |
| 207 def CopyFromCache(options, filename, dest_filename): | |
| 208 """Copy an item from the cache. | |
| 209 | |
| 210 Args: | |
| 211 options: bag of options. | |
| 212 filename: filename relative to the top of the download url / cache. | |
| 213 dest_filename: location to copy the file to. | |
| 214 """ | |
| 215 dpath = CachedPath(options, filename) | |
| 216 shutil.copy(dpath, dest_filename) | |
| 217 assert Sha1Sum(dest_filename) == Sha1FromFilename(filename) | |
| 218 | |
| 219 | |
| 220 def TestValidators(options, work_dir): | |
| 221 """Test x86 validators on current snapshot. | |
| 222 | |
| 223 Args: | |
| 224 options: bag of options. | |
| 225 work_dir: directory to operate in. | |
| 226 """ | |
| 227 list_filename = os.path.join(work_dir, 'naclapps.list') | |
| 228 nexe_filename = os.path.join(work_dir, 'test.nexe') | |
| 229 DownloadNexeList(list_filename) | |
| 230 fh = open(list_filename) | |
| 231 filenames = fh.read().splitlines() | |
| 232 fh.close() | |
| 233 failures = 0 | |
| 234 successes = 0 | |
| 235 start = time.time() | |
| 236 count = len(filenames) | |
| 237 for index, filename in enumerate(filenames): | |
| 238 tm = time.time() | |
| 239 if index > 0: | |
| 240 eta = (count - index) * (tm - start) / index | |
| 241 eta_minutes = int(eta / 60) | |
| 242 eta_seconds = int(eta - eta_minutes * 60) | |
| 243 eta_str = ' (ETA %d:%02d)' % (eta_minutes, eta_seconds) | |
| 244 else: | |
| 245 eta_str = '' | |
| 246 print 'Processing %d of %d%s...' % (index + 1, count, eta_str) | |
| 247 PrimeCache(options, filename) | |
| 248 # Stop here if downloading only. | |
| 249 if options.download_only: | |
| 250 continue | |
| 251 # Skip if not the right architecture. | |
| 252 architecture = NexeArchitecture(CachedPath(options, filename)) | |
| 253 if architecture != options.architecture: | |
| 254 continue | |
| 255 # Validate a copy in case validator is mutating. | |
| 256 CopyFromCache(options, filename, nexe_filename) | |
| 257 try: | |
| 258 result = ValidateNexe( | |
| 259 options, nexe_filename, filename, | |
| 260 NexeShouldValidate(filename), failures < 1) | |
| 261 if result: | |
| 262 successes += 1 | |
| 263 else: | |
| 264 failures += 1 | |
| 265 if not result and not options.keep_going: | |
| 266 break | |
| 267 finally: | |
| 268 try: | |
| 269 os.remove(nexe_filename) | |
| 270 except OSError: | |
| 271 print 'ERROR - unable to remove %s' % nexe_filename | |
| 272 print 'Ran validator on %d of %d NEXEs' % ( | |
| 273 successes + failures, len(filenames)) | |
| 274 if failures: | |
| 275 # Our alternate validators don't currently cover everything. | |
| 276 # For now, don't fail just emit warning (and a tally of failures). | |
| 277 print '@@@STEP_WARNINGS@@@' | |
| 278 print '@@@STEP_TEXT@FAILED %d times (%.1f%% are incorrect)@@@' % ( | |
| 279 failures, failures * 100 / (successes + failures)) | |
| 280 else: | |
| 281 print 'SUCCESS' | |
| 282 | |
| 283 | |
| 284 def Main(): | |
| 285 # Decide a default cache directory. | |
| 286 # Prefer /b (for the bots) | |
| 287 # Failing that, use scons-out. | |
| 288 # Failing that, use the current users's home dir. | |
| 289 default_cache_dir = '/b' | |
| 290 if not os.path.isdir(default_cache_dir): | |
| 291 default_cache_dir = os.path.join(NACL_DIR, 'scons-out') | |
| 292 if not os.path.isdir(default_cache_dir): | |
| 293 default_cache_dir = os.path.expanduser('~/') | |
| 294 default_cache_dir = os.path.abspath(default_cache_dir) | |
| 295 assert os.path.isdir(default_cache_dir) | |
| 296 | |
| 297 parser = optparse.OptionParser() | |
| 298 parser.add_option( | |
| 299 '--cache-dir', dest='cache_dir', default=default_cache_dir, | |
| 300 help='directory to cache downloads in') | |
| 301 parser.add_option( | |
| 302 '--download-only', dest='download_only', | |
| 303 default=False, action='store_true', | |
| 304 help='download to cache without running the tests') | |
| 305 parser.add_option( | |
| 306 '-k', '--keep-going', dest='keep_going', | |
| 307 default=False, action='store_true', | |
| 308 help='keep going on failure') | |
| 309 parser.add_option( | |
| 310 '--validator', dest='validator', | |
| 311 help='location of validator executable') | |
| 312 parser.add_option( | |
| 313 '--arch', dest='architecture', | |
| 314 help='architecture of validator') | |
| 315 options, args = parser.parse_args() | |
| 316 if args: | |
| 317 parser.error('unused arguments') | |
| 318 if not options.download_only: | |
| 319 if not options.validator: | |
| 320 parser.error('no validator specified') | |
| 321 if not options.architecture: | |
| 322 parser.error('no architecture specified') | |
| 323 | |
| 324 work_dir = tempfile.mkdtemp(suffix='validate_nexes', prefix='tmp') | |
| 325 try: | |
| 326 TestValidators(options, work_dir) | |
| 327 finally: | |
| 328 download_utils.RemoveDir(work_dir) | |
| 329 | |
| 330 | |
| 331 if __name__ == '__main__': | |
| 332 Main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 9950041:
Adding a startup regression test. (Closed)
Base URL: svn://svn.chromium.org/native_client/trunk/src/native_client