| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- test-case-name: buildbot.test.test_transfer -*- | |
| 2 | |
| 3 import os.path, tarfile, tempfile | |
| 4 from twisted.internet import reactor | |
| 5 from twisted.spread import pb | |
| 6 from twisted.python import log | |
| 7 from buildbot.process.buildstep import RemoteCommand, BuildStep | |
| 8 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED | |
| 9 from buildbot.interfaces import BuildSlaveTooOldError | |
| 10 | |
| 11 | |
| 12 class _FileWriter(pb.Referenceable): | |
| 13 """ | |
| 14 Helper class that acts as a file-object with write access | |
| 15 """ | |
| 16 | |
| 17 def __init__(self, destfile, maxsize, mode): | |
| 18 # Create missing directories. | |
| 19 destfile = os.path.abspath(destfile) | |
| 20 dirname = os.path.dirname(destfile) | |
| 21 if not os.path.exists(dirname): | |
| 22 os.makedirs(dirname) | |
| 23 | |
| 24 self.destfile = destfile | |
| 25 self.fp = open(destfile, "wb") | |
| 26 if mode is not None: | |
| 27 os.chmod(destfile, mode) | |
| 28 self.remaining = maxsize | |
| 29 | |
| 30 def remote_write(self, data): | |
| 31 """ | |
| 32 Called from remote slave to write L{data} to L{fp} within boundaries | |
| 33 of L{maxsize} | |
| 34 | |
| 35 @type data: C{string} | |
| 36 @param data: String of data to write | |
| 37 """ | |
| 38 if self.remaining is not None: | |
| 39 if len(data) > self.remaining: | |
| 40 data = data[:self.remaining] | |
| 41 self.fp.write(data) | |
| 42 self.remaining = self.remaining - len(data) | |
| 43 else: | |
| 44 self.fp.write(data) | |
| 45 | |
| 46 def remote_close(self): | |
| 47 """ | |
| 48 Called by remote slave to state that no more data will be transfered | |
| 49 """ | |
| 50 self.fp.close() | |
| 51 self.fp = None | |
| 52 | |
| 53 def __del__(self): | |
| 54 # unclean shutdown, the file is probably truncated, so delete it | |
| 55 # altogether rather than deliver a corrupted file | |
| 56 fp = getattr(self, "fp", None) | |
| 57 if fp: | |
| 58 fp.close() | |
| 59 os.unlink(self.destfile) | |
| 60 | |
| 61 | |
| 62 def _extractall(self, path=".", members=None): | |
| 63 """Fallback extractall method for TarFile, in case it doesn't have its own."
"" | |
| 64 | |
| 65 import copy | |
| 66 import operator | |
| 67 | |
| 68 directories = [] | |
| 69 | |
| 70 if members is None: | |
| 71 members = self | |
| 72 | |
| 73 for tarinfo in members: | |
| 74 if tarinfo.isdir(): | |
| 75 # Extract directories with a safe mode. | |
| 76 directories.append(tarinfo) | |
| 77 tarinfo = copy.copy(tarinfo) | |
| 78 tarinfo.mode = 0700 | |
| 79 self.extract(tarinfo, path) | |
| 80 | |
| 81 # Reverse sort directories. | |
| 82 directories.sort(lambda a, b: cmp(a.name, b.name)) | |
| 83 directories.reverse() | |
| 84 | |
| 85 # Set correct owner, mtime and filemode on directories. | |
| 86 for tarinfo in directories: | |
| 87 dirpath = os.path.join(path, tarinfo.name) | |
| 88 try: | |
| 89 self.chown(tarinfo, dirpath) | |
| 90 self.utime(tarinfo, dirpath) | |
| 91 self.chmod(tarinfo, dirpath) | |
| 92 except tarfile.ExtractError, e: | |
| 93 if self.errorlevel > 1: | |
| 94 raise | |
| 95 else: | |
| 96 self._dbg(1, "tarfile: %s" % e) | |
| 97 | |
| 98 class _DirectoryWriter(_FileWriter): | |
| 99 """ | |
| 100 A DirectoryWriter is implemented as a FileWriter, with an added post-process
ing | |
| 101 step to unpack the archive, once the transfer has completed. | |
| 102 """ | |
| 103 | |
| 104 def __init__(self, destroot, maxsize, compress, mode): | |
| 105 self.destroot = destroot | |
| 106 | |
| 107 self.fd, self.tarname = tempfile.mkstemp() | |
| 108 self.compress = compress | |
| 109 _FileWriter.__init__(self, self.tarname, maxsize, mode) | |
| 110 | |
| 111 def remote_unpack(self): | |
| 112 """ | |
| 113 Called by remote slave to state that no more data will be transfered | |
| 114 """ | |
| 115 if self.fp: | |
| 116 self.fp.close() | |
| 117 self.fp = None | |
| 118 fileobj = os.fdopen(self.fd, 'r') | |
| 119 if self.compress == 'bz2': | |
| 120 mode='r|bz2' | |
| 121 elif self.compress == 'gz': | |
| 122 mode='r|gz' | |
| 123 else: | |
| 124 mode = 'r' | |
| 125 if not hasattr(tarfile.TarFile, 'extractall'): | |
| 126 tarfile.TarFile.extractall = _extractall | |
| 127 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) | |
| 128 archive.extractall(path=self.destroot) | |
| 129 os.remove(self.tarname) | |
| 130 | |
| 131 | |
| 132 class StatusRemoteCommand(RemoteCommand): | |
| 133 def __init__(self, remote_command, args): | |
| 134 RemoteCommand.__init__(self, remote_command, args) | |
| 135 | |
| 136 self.rc = None | |
| 137 self.stderr = '' | |
| 138 | |
| 139 def remoteUpdate(self, update): | |
| 140 #log.msg('StatusRemoteCommand: update=%r' % update) | |
| 141 if 'rc' in update: | |
| 142 self.rc = update['rc'] | |
| 143 if 'stderr' in update: | |
| 144 self.stderr = self.stderr + update['stderr'] + '\n' | |
| 145 | |
| 146 class _TransferBuildStep(BuildStep): | |
| 147 """ | |
| 148 Base class for FileUpload and FileDownload to factor out common | |
| 149 functionality. | |
| 150 """ | |
| 151 DEFAULT_WORKDIR = "build" # is this redundant? | |
| 152 | |
| 153 def setDefaultWorkdir(self, workdir): | |
| 154 if self.workdir is None: | |
| 155 self.workdir = workdir | |
| 156 | |
| 157 def _getWorkdir(self): | |
| 158 properties = self.build.getProperties() | |
| 159 if self.workdir is None: | |
| 160 workdir = self.DEFAULT_WORKDIR | |
| 161 else: | |
| 162 workdir = self.workdir | |
| 163 return properties.render(workdir) | |
| 164 | |
| 165 def finished(self, result): | |
| 166 # Subclasses may choose to skip a transfer. In those cases, self.cmd | |
| 167 # will be None, and we should just let BuildStep.finished() handle | |
| 168 # the rest | |
| 169 if result == SKIPPED: | |
| 170 return BuildStep.finished(self, SKIPPED) | |
| 171 if self.cmd.stderr != '': | |
| 172 self.addCompleteLog('stderr', self.cmd.stderr) | |
| 173 | |
| 174 if self.cmd.rc is None or self.cmd.rc == 0: | |
| 175 return BuildStep.finished(self, SUCCESS) | |
| 176 return BuildStep.finished(self, FAILURE) | |
| 177 | |
| 178 | |
| 179 class FileUpload(_TransferBuildStep): | |
| 180 """ | |
| 181 Build step to transfer a file from the slave to the master. | |
| 182 | |
| 183 arguments: | |
| 184 | |
| 185 - ['slavesrc'] filename of source file at slave, relative to workdir | |
| 186 - ['masterdest'] filename of destination file at master | |
| 187 - ['workdir'] string with slave working directory relative to builder | |
| 188 base dir, default 'build' | |
| 189 - ['maxsize'] maximum size of the file, default None (=unlimited) | |
| 190 - ['blocksize'] maximum size of each block being transfered | |
| 191 - ['mode'] file access mode for the resulting master-side file. | |
| 192 The default (=None) is to leave it up to the umask of | |
| 193 the buildmaster process. | |
| 194 | |
| 195 """ | |
| 196 | |
| 197 name = 'upload' | |
| 198 | |
| 199 def __init__(self, slavesrc, masterdest, | |
| 200 workdir=None, maxsize=None, blocksize=16*1024, mode=None, | |
| 201 **buildstep_kwargs): | |
| 202 BuildStep.__init__(self, **buildstep_kwargs) | |
| 203 self.addFactoryArguments(slavesrc=slavesrc, | |
| 204 masterdest=masterdest, | |
| 205 workdir=workdir, | |
| 206 maxsize=maxsize, | |
| 207 blocksize=blocksize, | |
| 208 mode=mode, | |
| 209 ) | |
| 210 | |
| 211 self.slavesrc = slavesrc | |
| 212 self.masterdest = masterdest | |
| 213 self.workdir = workdir | |
| 214 self.maxsize = maxsize | |
| 215 self.blocksize = blocksize | |
| 216 assert isinstance(mode, (int, type(None))) | |
| 217 self.mode = mode | |
| 218 | |
| 219 def start(self): | |
| 220 version = self.slaveVersion("uploadFile") | |
| 221 properties = self.build.getProperties() | |
| 222 | |
| 223 if not version: | |
| 224 m = "slave is too old, does not know about uploadFile" | |
| 225 raise BuildSlaveTooOldError(m) | |
| 226 | |
| 227 source = properties.render(self.slavesrc) | |
| 228 masterdest = properties.render(self.masterdest) | |
| 229 # we rely upon the fact that the buildmaster runs chdir'ed into its | |
| 230 # basedir to make sure that relative paths in masterdest are expanded | |
| 231 # properly. TODO: maybe pass the master's basedir all the way down | |
| 232 # into the BuildStep so we can do this better. | |
| 233 masterdest = os.path.expanduser(masterdest) | |
| 234 log.msg("FileUpload started, from slave %r to master %r" | |
| 235 % (source, masterdest)) | |
| 236 | |
| 237 self.step_status.setText(['uploading', os.path.basename(source)]) | |
| 238 | |
| 239 # we use maxsize to limit the amount of data on both sides | |
| 240 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) | |
| 241 | |
| 242 # default arguments | |
| 243 args = { | |
| 244 'slavesrc': source, | |
| 245 'workdir': self._getWorkdir(), | |
| 246 'writer': fileWriter, | |
| 247 'maxsize': self.maxsize, | |
| 248 'blocksize': self.blocksize, | |
| 249 } | |
| 250 | |
| 251 self.cmd = StatusRemoteCommand('uploadFile', args) | |
| 252 d = self.runCommand(self.cmd) | |
| 253 d.addCallback(self.finished).addErrback(self.failed) | |
| 254 | |
| 255 | |
| 256 class DirectoryUpload(BuildStep): | |
| 257 """ | |
| 258 Build step to transfer a directory from the slave to the master. | |
| 259 | |
| 260 arguments: | |
| 261 | |
| 262 - ['slavesrc'] name of source directory at slave, relative to workdir | |
| 263 - ['masterdest'] name of destination directory at master | |
| 264 - ['workdir'] string with slave working directory relative to builder | |
| 265 base dir, default 'build' | |
| 266 - ['maxsize'] maximum size of the compressed tarfile containing the | |
| 267 whole directory | |
| 268 - ['blocksize'] maximum size of each block being transfered | |
| 269 - ['compress'] compression type to use: one of [None, 'gz', 'bz2'] | |
| 270 | |
| 271 """ | |
| 272 | |
| 273 name = 'upload' | |
| 274 | |
| 275 def __init__(self, slavesrc, masterdest, | |
| 276 workdir="build", maxsize=None, blocksize=16*1024, | |
| 277 compress=None, **buildstep_kwargs): | |
| 278 BuildStep.__init__(self, **buildstep_kwargs) | |
| 279 self.addFactoryArguments(slavesrc=slavesrc, | |
| 280 masterdest=masterdest, | |
| 281 workdir=workdir, | |
| 282 maxsize=maxsize, | |
| 283 blocksize=blocksize, | |
| 284 compress=compress, | |
| 285 ) | |
| 286 | |
| 287 self.slavesrc = slavesrc | |
| 288 self.masterdest = masterdest | |
| 289 self.workdir = workdir | |
| 290 self.maxsize = maxsize | |
| 291 self.blocksize = blocksize | |
| 292 assert compress in (None, 'gz', 'bz2') | |
| 293 self.compress = compress | |
| 294 | |
| 295 def start(self): | |
| 296 version = self.slaveVersion("uploadDirectory") | |
| 297 properties = self.build.getProperties() | |
| 298 | |
| 299 if not version: | |
| 300 m = "slave is too old, does not know about uploadDirectory" | |
| 301 raise BuildSlaveTooOldError(m) | |
| 302 | |
| 303 source = properties.render(self.slavesrc) | |
| 304 masterdest = properties.render(self.masterdest) | |
| 305 # we rely upon the fact that the buildmaster runs chdir'ed into its | |
| 306 # basedir to make sure that relative paths in masterdest are expanded | |
| 307 # properly. TODO: maybe pass the master's basedir all the way down | |
| 308 # into the BuildStep so we can do this better. | |
| 309 masterdest = os.path.expanduser(masterdest) | |
| 310 log.msg("DirectoryUpload started, from slave %r to master %r" | |
| 311 % (source, masterdest)) | |
| 312 | |
| 313 self.step_status.setText(['uploading', os.path.basename(source)]) | |
| 314 | |
| 315 # we use maxsize to limit the amount of data on both sides | |
| 316 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, 06
00) | |
| 317 | |
| 318 # default arguments | |
| 319 args = { | |
| 320 'slavesrc': source, | |
| 321 'workdir': self.workdir, | |
| 322 'writer': dirWriter, | |
| 323 'maxsize': self.maxsize, | |
| 324 'blocksize': self.blocksize, | |
| 325 'compress': self.compress | |
| 326 } | |
| 327 | |
| 328 self.cmd = StatusRemoteCommand('uploadDirectory', args) | |
| 329 d = self.runCommand(self.cmd) | |
| 330 d.addCallback(self.finished).addErrback(self.failed) | |
| 331 | |
| 332 def finished(self, result): | |
| 333 # Subclasses may choose to skip a transfer. In those cases, self.cmd | |
| 334 # will be None, and we should just let BuildStep.finished() handle | |
| 335 # the rest | |
| 336 if result == SKIPPED: | |
| 337 return BuildStep.finished(self, SKIPPED) | |
| 338 if self.cmd.stderr != '': | |
| 339 self.addCompleteLog('stderr', self.cmd.stderr) | |
| 340 | |
| 341 if self.cmd.rc is None or self.cmd.rc == 0: | |
| 342 return BuildStep.finished(self, SUCCESS) | |
| 343 return BuildStep.finished(self, FAILURE) | |
| 344 | |
| 345 | |
| 346 | |
| 347 | |
| 348 class _FileReader(pb.Referenceable): | |
| 349 """ | |
| 350 Helper class that acts as a file-object with read access | |
| 351 """ | |
| 352 | |
| 353 def __init__(self, fp): | |
| 354 self.fp = fp | |
| 355 | |
| 356 def remote_read(self, maxlength): | |
| 357 """ | |
| 358 Called from remote slave to read at most L{maxlength} bytes of data | |
| 359 | |
| 360 @type maxlength: C{integer} | |
| 361 @param maxlength: Maximum number of data bytes that can be returned | |
| 362 | |
| 363 @return: Data read from L{fp} | |
| 364 @rtype: C{string} of bytes read from file | |
| 365 """ | |
| 366 if self.fp is None: | |
| 367 return '' | |
| 368 | |
| 369 data = self.fp.read(maxlength) | |
| 370 return data | |
| 371 | |
| 372 def remote_close(self): | |
| 373 """ | |
| 374 Called by remote slave to state that no more data will be transfered | |
| 375 """ | |
| 376 if self.fp is not None: | |
| 377 self.fp.close() | |
| 378 self.fp = None | |
| 379 | |
| 380 | |
| 381 class FileDownload(_TransferBuildStep): | |
| 382 """ | |
| 383 Download the first 'maxsize' bytes of a file, from the buildmaster to the | |
| 384 buildslave. Set the mode of the file | |
| 385 | |
| 386 Arguments:: | |
| 387 | |
| 388 ['mastersrc'] filename of source file at master | |
| 389 ['slavedest'] filename of destination file at slave | |
| 390 ['workdir'] string with slave working directory relative to builder | |
| 391 base dir, default 'build' | |
| 392 ['maxsize'] maximum size of the file, default None (=unlimited) | |
| 393 ['blocksize'] maximum size of each block being transfered | |
| 394 ['mode'] use this to set the access permissions of the resulting | |
| 395 buildslave-side file. This is traditionally an octal | |
| 396 integer, like 0644 to be world-readable (but not | |
| 397 world-writable), or 0600 to only be readable by | |
| 398 the buildslave account, or 0755 to be world-executable. | |
| 399 The default (=None) is to leave it up to the umask of | |
| 400 the buildslave process. | |
| 401 | |
| 402 """ | |
| 403 name = 'download' | |
| 404 | |
| 405 def __init__(self, mastersrc, slavedest, | |
| 406 workdir=None, maxsize=None, blocksize=16*1024, mode=None, | |
| 407 **buildstep_kwargs): | |
| 408 BuildStep.__init__(self, **buildstep_kwargs) | |
| 409 self.addFactoryArguments(mastersrc=mastersrc, | |
| 410 slavedest=slavedest, | |
| 411 workdir=workdir, | |
| 412 maxsize=maxsize, | |
| 413 blocksize=blocksize, | |
| 414 mode=mode, | |
| 415 ) | |
| 416 | |
| 417 self.mastersrc = mastersrc | |
| 418 self.slavedest = slavedest | |
| 419 self.workdir = workdir | |
| 420 self.maxsize = maxsize | |
| 421 self.blocksize = blocksize | |
| 422 assert isinstance(mode, (int, type(None))) | |
| 423 self.mode = mode | |
| 424 | |
| 425 def start(self): | |
| 426 properties = self.build.getProperties() | |
| 427 | |
| 428 version = self.slaveVersion("downloadFile") | |
| 429 if not version: | |
| 430 m = "slave is too old, does not know about downloadFile" | |
| 431 raise BuildSlaveTooOldError(m) | |
| 432 | |
| 433 # we are currently in the buildmaster's basedir, so any non-absolute | |
| 434 # paths will be interpreted relative to that | |
| 435 source = os.path.expanduser(properties.render(self.mastersrc)) | |
| 436 slavedest = properties.render(self.slavedest) | |
| 437 log.msg("FileDownload started, from master %r to slave %r" % | |
| 438 (source, slavedest)) | |
| 439 | |
| 440 self.step_status.setText(['downloading', "to", | |
| 441 os.path.basename(slavedest)]) | |
| 442 | |
| 443 # setup structures for reading the file | |
| 444 try: | |
| 445 fp = open(source, 'rb') | |
| 446 except IOError: | |
| 447 # if file does not exist, bail out with an error | |
| 448 self.addCompleteLog('stderr', | |
| 449 'File %r not available at master' % source) | |
| 450 # TODO: once BuildStep.start() gets rewritten to use | |
| 451 # maybeDeferred, just re-raise the exception here. | |
| 452 reactor.callLater(0, BuildStep.finished, self, FAILURE) | |
| 453 return | |
| 454 fileReader = _FileReader(fp) | |
| 455 | |
| 456 # default arguments | |
| 457 args = { | |
| 458 'slavedest': slavedest, | |
| 459 'maxsize': self.maxsize, | |
| 460 'reader': fileReader, | |
| 461 'blocksize': self.blocksize, | |
| 462 'workdir': self._getWorkdir(), | |
| 463 'mode': self.mode, | |
| 464 } | |
| 465 | |
| 466 self.cmd = StatusRemoteCommand('downloadFile', args) | |
| 467 d = self.runCommand(self.cmd) | |
| 468 d.addCallback(self.finished).addErrback(self.failed) | |
| 469 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12207158:
Bye bye buildbot 0.7.12. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build