swarming: custom cipd package paths

client/isolateserver.py

 #!/usr/bin/env python
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Archives a set of files or directories to an Isolate Server."""
 
 __version__ = '0.4.8'
 
 import base64
@@ skipping 98 matching lines @@
 class Error(Exception):
   """Generic runtime error."""
   pass
 
 
 class Aborted(Error):
   """Operation aborted."""
   pass
 
 
+class AlreadyExists(Error):
+  """File already exists."""
+
+
 def file_read(path, chunk_size=isolated_format.DISK_FILE_CHUNK, offset=0):
   """Yields file content in chunks of |chunk_size| starting from |offset|."""
   with fs.open(path, 'rb') as f:
     if offset:
       f.seek(offset)
     while True:
       data = f.read(chunk_size)
       if not data:
         break
       yield data
@@ skipping 85 matching lines @@
 def create_symlinks(base_directory, files):
   """Creates any symlinks needed by the given set of files."""
   for filepath, properties in files:
     if 'l' not in properties:
       continue
     if sys.platform == 'win32':
       # TODO(maruel): Create symlink via the win32 api.
       logging.warning('Ignoring symlink %s', filepath)
       continue
     outfile = os.path.join(base_directory, filepath)
+    if os.path.exists(outfile):

[M-A Ruel, 2016/06/15 17:28:27]

I'm not super excited about adding another system call here, could we trap the
exception generated by os.symlink()?

import errno

try:
  os.symlink(...)
except OSError as e:
  if e.errno == errno.EEXIST:
    raise AlreadyExists(...)
  raise

tested to work on both Ubuntu and OSX

[nodir, 2016/06/15 17:53:42]

Done

+      raise AlreadyExists('File %s already exists.' % outfile)
     # os.symlink() doesn't exist on Windows.
     os.symlink(properties['l'], outfile)  # pylint: disable=E1101
 
 
 def is_valid_file(path, size):
   """Determines if the given files appears valid.
 
   Currently it just checks the file's size.
   """
   if size == UNKNOWN_FILE_SIZE:
@@ skipping 1726 matching lines @@
       last_update = time.time()
       with threading_utils.DeadlockDetector(DEADLOCK_TIMEOUT) as detector:
         while remaining:
           detector.ping()
 
           # Wait for any item to finish fetching to cache.
           digest = fetch_queue.wait(remaining)
 
           # Link corresponding files to a fetched item in cache.
           for filepath, props in remaining.pop(digest):
-            cache.hardlink(
-                digest, os.path.join(outdir, filepath), props.get('m'))
+            dest = os.path.join(outdir, filepath)
+            if os.path.exists(dest):

[M-A Ruel, 2016/06/15 17:28:27]

this one is harder to skip since the actual execution is implementation
dependent.

[nodir, 2016/06/15 17:53:42]

Acknowledged.

+              raise AlreadyExists('File %s already exists' % dest)
+            cache.hardlink(digest, dest, props.get('m'))
 
           # Report progress.
           duration = time.time() - last_update
           if duration > DELAY_BETWEEN_UPDATES_IN_SECS:
             msg = '%d files remaining...' % len(remaining)
             print msg
             logging.info(msg)
             last_update = time.time()
 
   # Cache could evict some items we just tried to fetch, it's a fatal error.
@@ skipping 316 matching lines @@
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParserIsolateServer(), args)
 
 
 if __name__ == '__main__':
   subprocess42.inhibit_os_error_reporting()
   fix_encoding.fix_encoding()
   tools.disable_buffering()
   colorama.init()
   sys.exit(main(sys.argv[1:]))