Shuffle code around in file_path.

client/utils/file_path.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Provides functions: get_native_path_case(), isabs() and safe_join().
 
 This module assumes that filesystem is not changing while current process
 is running and thus it caches results of functions that depend on FS state.
 """
 
@@ skipping 228 matching lines @@
     # its group or that the current ACL permits this. Otherwise it will silently
     # fail.
     win32security.SetFileSecurity(
         fs.extend(path), win32security.DACL_SECURITY_INFORMATION, sd)
     # It's important to also look for the read only bit after, as it's possible
     # the set_read_only() call to remove the read only bit had silently failed
     # because there was no DACL for the user.
     if not (os.stat(path).st_mode & stat.S_IWUSR):
       os.chmod(path, 0777)
 
+
   def isabs(path):
     """Accepts X: as an absolute path, unlike python's os.path.isabs()."""
     return os.path.isabs(path) or len(path) == 2 and path[1] == ':'
 
 
   def find_item_native_case(root, item):
     """Gets the native path case of a single item based at root_path."""
     if item == '..':
       return item
 
@@ skipping 41 matching lines @@
         # The path does not exist. Try to recurse and reconstruct the path.
         base = os.path.dirname(p)
         rest = os.path.basename(p)
         return os.path.join(get_native_path_case(base), rest)
       raise
     # Always upper case the first letter since GetLongPathName() will return the
     # drive letter in the case it was given.
     return out[0].upper() + out[1:] + suffix
 
 
-  def enum_processes_win():
+  def get_process_token():
+    """Get the current process token."""
+    TOKEN_ALL_ACCESS = 0xF01FF
+    token = ctypes.wintypes.HANDLE()
+    if not OpenProcessToken(
+        GetCurrentProcess(), TOKEN_ALL_ACCESS, ctypes.byref(token)):
+      # pylint: disable=undefined-variable
+      raise WindowsError('Couldn\'t get process token')
+    return token
+
+
+  def get_luid(name):
+    """Returns the LUID for a privilege."""
+    luid = LUID()
+    if not LookupPrivilegeValue(None, unicode(name), ctypes.byref(luid)):
+      # pylint: disable=undefined-variable
+      raise WindowsError('Couldn\'t lookup privilege value')
+    return luid
+
+
+  def enable_privilege(name):
+    """Enables the privilege for the current process token.
+
+    Returns:
+    - True if the assignment is successful.
+    """
+    SE_PRIVILEGE_ENABLED = 2
+    ERROR_NOT_ALL_ASSIGNED = 1300
+
+    size = ctypes.sizeof(TOKEN_PRIVILEGES) + ctypes.sizeof(LUID_AND_ATTRIBUTES)
+    buf = ctypes.create_string_buffer(size)
+    tp = ctypes.cast(buf, ctypes.POINTER(TOKEN_PRIVILEGES)).contents
+    tp.count = 1
+    tp.get_array()[0].LUID = get_luid(name)
+    tp.get_array()[0].Attributes = SE_PRIVILEGE_ENABLED
+    token = get_process_token()
+    try:
+      if not AdjustTokenPrivileges(token, False, tp, 0, None, None):
+        # pylint: disable=undefined-variable
+        raise WindowsError('Error in AdjustTokenPrivileges')
+    finally:
+      ctypes.windll.kernel32.CloseHandle(token)
+    return ctypes.windll.kernel32.GetLastError() != ERROR_NOT_ALL_ASSIGNED
+
+
+  def enable_symlink():
+    """Enable SeCreateSymbolicLinkPrivilege for the current token.
+
+    Returns:
+    - True if symlink support is enabled.
+
+    Thanks Microsoft. This is appreciated.
+    """
+    return enable_privilege(u'SeCreateSymbolicLinkPrivilege')
+
+
+  def kill_children_processes(root):
+    """Try to kill all children processes indistriminately and prints updates to
+    stderr.
+
+    Returns:
+      True if at least one child process was found.
+    """
+    processes = _get_children_processes_win(root)
+    if not processes:
+      return False
+    sys.stderr.write('Enumerating processes:\n')
+    for _, proc in sorted(processes.iteritems()):
+      sys.stderr.write(
+          '- pid %d; Handles: %d; Exe: %s; Cmd: %s\n' % (
+            proc.ProcessId,
+            proc.HandleCount,
+            proc.ExecutablePath,
+            proc.CommandLine))
+    sys.stderr.write('Terminating %d processes:\n' % len(processes))
+    for pid in sorted(processes):
+      try:
+        # Killing is asynchronous.
+        os.kill(pid, 9)
+        sys.stderr.write('- %d killed\n' % pid)
+      except OSError:
+        sys.stderr.write('- failed to kill %s\n' % pid)
+    return True
+
+
+  ## Windows private code.
+
+
+  def _enum_processes_win():
     """Returns all processes on the system that are accessible to this process.
 
     Returns:
       Win32_Process COM objects. See
       http://msdn.microsoft.com/library/aa394372.aspx for more details.
     """
     import win32com.client  # pylint: disable=F0401
     wmi_service = win32com.client.Dispatch('WbemScripting.SWbemLocator')
     wbem = wmi_service.ConnectServer('.', 'root\\cimv2')
     return [proc for proc in wbem.ExecQuery('SELECT * FROM Win32_Process')]
 
 
-  def filter_processes_dir_win(processes, root_dir):
+  def _filter_processes_dir_win(processes, root_dir):
     """Returns all processes which has their main executable located inside
     root_dir.
     """
     def normalize_path(filename):
       try:
         return GetLongPathName(unicode(filename)).lower()
       except:  # pylint: disable=W0702
         return unicode(filename).lower()
 
     root_dir = normalize_path(root_dir)
@@ skipping 11 matching lines @@
       return None
 
     long_names = ((process_name(proc), proc) for proc in processes)
 
     return [
       proc for name, proc in long_names
       if name is not None and name.startswith(root_dir)
     ]
 
 
-  def filter_processes_tree_win(processes):
+  def _filter_processes_tree_win(processes):
     """Returns all the processes under the current process."""
     # Convert to dict.
     processes = dict((p.ProcessId, p) for p in processes)
     root_pid = os.getpid()
     out = {root_pid: processes[root_pid]}
     while True:
       found = set()
       for pid in out:
         found.update(
             p.ProcessId for p in processes.itervalues()
             if p.ParentProcessId == pid)
       found -= set(out)
       if not found:
         break
       out.update((p, processes[p]) for p in found)
     return out.values()
 
 
-  def get_process_token():
-    """Get the current process token."""
-    TOKEN_ALL_ACCESS = 0xF01FF
-    token = ctypes.wintypes.HANDLE()
-    if not OpenProcessToken(
-        GetCurrentProcess(), TOKEN_ALL_ACCESS, ctypes.byref(token)):
-      # pylint: disable=undefined-variable
-      raise WindowsError('Couldn\'t get process token')
-    return token
+  def _get_children_processes_win(root):
+    """Returns a list of processes.
 
+    Enumerates both:
+    - all child processes from this process.
+    - processes where the main executable in inside 'root'. The reason is that
+      the ancestry may be broken so stray grand-children processes could be
+      undetected by the first technique.
 
-  def get_luid(name):
-    """Returns the LUID for a privilege."""
-    luid = LUID()
-    if not LookupPrivilegeValue(None, unicode(name), ctypes.byref(luid)):
-      # pylint: disable=undefined-variable
-      raise WindowsError('Couldn\'t lookup privilege value')
-    return luid
-
-
-  def enable_privilege(name):
-    """Enables the privilege for the current process token.
-
-    Returns:
-    - True if the assignment is successful.
+    This technique is not fool-proof but gets mostly there.
     """
-    SE_PRIVILEGE_ENABLED = 2
-    ERROR_NOT_ALL_ASSIGNED = 1300
-
-    size = ctypes.sizeof(TOKEN_PRIVILEGES) + ctypes.sizeof(LUID_AND_ATTRIBUTES)
-    buf = ctypes.create_string_buffer(size)
-    tp = ctypes.cast(buf, ctypes.POINTER(TOKEN_PRIVILEGES)).contents
-    tp.count = 1
-    tp.get_array()[0].LUID = get_luid(name)
-    tp.get_array()[0].Attributes = SE_PRIVILEGE_ENABLED
-    token = get_process_token()
-    try:
-      if not AdjustTokenPrivileges(token, False, tp, 0, None, None):
-        # pylint: disable=undefined-variable
-        raise WindowsError('Error in AdjustTokenPrivileges')
-    finally:
-      ctypes.windll.kernel32.CloseHandle(token)
-    return ctypes.windll.kernel32.GetLastError() != ERROR_NOT_ALL_ASSIGNED
-
-
-  def enable_symlink():
-    """Enable SeCreateSymbolicLinkPrivilege for the current token.
-
-    Returns:
-    - True if symlink support is enabled.
-
-    Thanks Microsoft. This is appreciated.
-    """
-    return enable_privilege(u'SeCreateSymbolicLinkPrivilege')
+    processes = _enum_processes_win()
+    tree_processes = _filter_processes_tree_win(processes)
+    dir_processes = _filter_processes_dir_win(processes, root)
+    # Convert to dict to remove duplicates.
+    processes = dict((p.ProcessId, p) for p in tree_processes)
+    processes.update((p.ProcessId, p) for p in dir_processes)
+    processes.pop(os.getpid())
+    return processes
 
 
 elif sys.platform == 'darwin':
 
 
   # On non-windows, keep the stdlib behavior.
   isabs = os.path.isabs
 
 
-  def _native_case(p):
-    """Gets the native path case. Warning: this function resolves symlinks."""
-    try:
-      rel_ref, _ = Carbon.File.FSPathMakeRef(p.encode('utf-8'))
-      # The OSX underlying code uses NFD but python strings are in NFC. This
-      # will cause issues with os.listdir() for example. Since the dtrace log
-      # *is* in NFC, normalize it here.
-      out = unicodedata.normalize(
-          'NFC', rel_ref.FSRefMakePath().decode('utf-8'))
-      if p.endswith(os.path.sep) and not out.endswith(os.path.sep):
-        return out + os.path.sep
-      return out
-    except MacOS.Error, e:
-      if e.args[0] in (-43, -120):
-        # The path does not exist. Try to recurse and reconstruct the path.
-        # -43 means file not found.
-        # -120 means directory not found.
-        base = os.path.dirname(p)
-        rest = os.path.basename(p)
-        return os.path.join(_native_case(base), rest)
-      raise OSError(
-          e.args[0], 'Failed to get native path for %s' % p, p, e.args[1])
-
-
-  def _split_at_symlink_native(base_path, rest):
-    """Returns the native path for a symlink."""
-    base, symlink, rest = split_at_symlink(base_path, rest)
-    if symlink:
-      if not base_path:
-        base_path = base
-      else:
-        base_path = safe_join(base_path, base)
-      symlink = find_item_native_case(base_path, symlink)
-    return base, symlink, rest
-
-
   def find_item_native_case(root_path, item):
     """Gets the native path case of a single item based at root_path.
 
     There is no API to get the native path case of symlinks on OSX. So it
     needs to be done the slow way.
     """
     if item == '..':
       return item
 
     item = item.lower()
@@ skipping 47 matching lines @@
     # Make sure no symlink was resolved.
     assert base.lower() == path.lower(), (base, path)
     logging.debug('get_native_path_case(%s) = %s' % (path, base))
     return base
 
 
   def enable_symlink():
     return True
 
 
+  ## OSX private code.
+
+
+  def _native_case(p):
+    """Gets the native path case. Warning: this function resolves symlinks."""
+    try:
+      rel_ref, _ = Carbon.File.FSPathMakeRef(p.encode('utf-8'))
+      # The OSX underlying code uses NFD but python strings are in NFC. This
+      # will cause issues with os.listdir() for example. Since the dtrace log
+      # *is* in NFC, normalize it here.
+      out = unicodedata.normalize(
+          'NFC', rel_ref.FSRefMakePath().decode('utf-8'))
+      if p.endswith(os.path.sep) and not out.endswith(os.path.sep):
+        return out + os.path.sep
+      return out
+    except MacOS.Error, e:
+      if e.args[0] in (-43, -120):
+        # The path does not exist. Try to recurse and reconstruct the path.
+        # -43 means file not found.
+        # -120 means directory not found.
+        base = os.path.dirname(p)
+        rest = os.path.basename(p)
+        return os.path.join(_native_case(base), rest)
+      raise OSError(
+          e.args[0], 'Failed to get native path for %s' % p, p, e.args[1])
+
+
+  def _split_at_symlink_native(base_path, rest):
+    """Returns the native path for a symlink."""
+    base, symlink, rest = split_at_symlink(base_path, rest)
+    if symlink:
+      if not base_path:
+        base_path = base
+      else:
+        base_path = safe_join(base_path, base)
+      symlink = find_item_native_case(base_path, symlink)
+    return base, symlink, rest
+
+
 else:  # OSes other than Windows and OSX.
 
 
   # On non-windows, keep the stdlib behavior.
   isabs = os.path.isabs
 
 
   def find_item_native_case(root, item):
     """Gets the native path case of a single item based at root_path."""
     if item == '..':
@@ skipping 95 matching lines @@
         logging.debug(
             'split_at_symlink(%s, %s) -> (%s, %s, %s)' %
             (base_dir, relfile, base, symlink, rest))
         return base, symlink, rest
       if index == len(relfile):
         break
       index += 1
     return relfile, None, None
 
 
+  def kill_children_processes(root):
+    """Not yet implemented on posix."""
+    # pylint: disable=unused-argument
+    return False
+
+
 def relpath(path, root):
   """os.path.relpath() that keeps trailing os.path.sep."""
   out = os.path.relpath(path, root)
   if path.endswith(os.path.sep):
     out += os.path.sep
   return out
 
 
 def safe_relpath(filepath, basepath):
   """Do not throw on Windows when filepath and basepath are on different drives.
@@ skipping 21 matching lines @@
   """posix.relpath() that keeps trailing slash.
 
   It is different from relpath() since it can be used on Windows.
   """
   out = posixpath.relpath(path, root)
   if path.endswith('/'):
     out += '/'
   return out
 
 
-def cleanup_path(x):
-  """Cleans up a relative path. Converts any os.path.sep to '/' on Windows."""
-  if x:
-    x = x.rstrip(os.path.sep).replace(os.path.sep, '/')
-  if x == '.':
-    x = ''
-  if x:
-    x += '/'
-  return x
-
-
 def is_url(path):
   """Returns True if it looks like an HTTP url instead of a file path."""
   return bool(re.match(r'^https?://.+$', path))
 
 
 def path_starts_with(prefix, path):
   """Returns true if the components of the path |prefix| are the same as the
   initial components of |path| (or all of the components of |path|). The paths
   must be absolute.
   """
@@ skipping 400 matching lines @@
   # The same path may be listed multiple times.
   for path in sorted(set(path for _, path, _ in errors)):
     sys.stderr.write('- %s\n' % path)
 
   # If soft retries fail on Linux, there's nothing better we can do.
   if sys.platform != 'win32':
     raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
 
   # The soft way was not good enough. Try the hard way.
   for i in xrange(max_tries):
-    if not _kill_children_processes_win(root):
+    if not kill_children_processes(root):
       break
     if i != max_tries - 1:
       time.sleep((i+1)*2)
   else:
     processes = _get_children_processes_win(root)
     if processes:
       sys.stderr.write('Failed to terminate processes.\n')
       raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
 
   # Now that annoying processes in root are evicted, try again.
   errors = []
   fs.rmtree(root, onerror=lambda *args: errors.append(args))
   if errors and fs.exists(root):
     # There's no hope: the directory was tried to be removed 4 times. Give up
     # and raise an exception.
     sys.stderr.write(
         'Failed to delete %s. The following files remain:\n' % root)
     # The same path may be listed multiple times.
     for path in sorted(set(path for _, path, _ in errors)):
       sys.stderr.write('- %s\n' % path)
     raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
   return False
 
 
 ## Private code.
-
-
-def _kill_children_processes_win(root):
-  """Try to kill all children processes indistriminately and prints updates to
-  stderr.
-
-  Returns:
-    True if at least one child process was found.
-  """
-  processes = _get_children_processes_win(root)
-  if not processes:
-    return False
-  sys.stderr.write('Enumerating processes:\n')
-  for _, proc in sorted(processes.iteritems()):
-    sys.stderr.write(
-        '- pid %d; Handles: %d; Exe: %s; Cmd: %s\n' % (
-          proc.ProcessId,
-          proc.HandleCount,
-          proc.ExecutablePath,
-          proc.CommandLine))
-  sys.stderr.write('Terminating %d processes:\n' % len(processes))
-  for pid in sorted(processes):
-    try:
-      # Killing is asynchronous.
-      os.kill(pid, 9)
-      sys.stderr.write('- %d killed\n' % pid)
-    except OSError:
-      sys.stderr.write('- failed to kill %s\n' % pid)
-  return True
-
-
-def _get_children_processes_win(root):
-  """Returns a list of processes.
-
-  Enumerates both:
-  - all child processes from this process.
-  - processes where the main executable in inside 'root'. The reason is that
-    the ancestry may be broken so stray grand-children processes could be
-    undetected by the first technique.
-
-  This technique is not fool-proof but gets mostly there.
-  """
-  processes = enum_processes_win()
-  tree_processes = filter_processes_tree_win(processes)
-  dir_processes = filter_processes_dir_win(processes, root)
-  # Convert to dict to remove duplicates.
-  processes = dict((p.ProcessId, p) for p in tree_processes)
-  processes.update((p.ProcessId, p) for p in dir_processes)
-  processes.pop(os.getpid())
-  return processes