[recipe_modules/file] add real file module to engine.

recipe_modules/file/api.py

+# Copyright 2017 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+from recipe_engine import recipe_api
+
+
+import os
+import fnmatch
+
+# TODO(iannucci): Introduce the concept of a 'native step' and implement these
+# directly in the current python interpreter without the need for a subprocess
+# invocation.
+
+class FileApi(recipe_api.RecipeApi):
+  """FileApi contains helper functions for reading and writing files."""
+
+  class Error(recipe_api.InfraFailure):
+    """Error is an InfraFailure, except that it also contains an errno field
+    indicating the errno name (i.e. 'EEXIST') of the underlying error.
+    """
+    def __init__(self, step_name, errno_name, message):
+      reason = 'Step(%r) failed %r with: %s' % (step_name, errno_name, message)
+      super(FileApi.Error, self).__init__(reason)
+      self.errno_name = errno_name
+
+  def _run(self, name, args, timeout, step_test_data=None, stdout=None):
+    if not step_test_data:
+      step_test_data = self.test_api.errno
+    args = [
+      '--json-output', self.m.json.output(name='main', add_json_log=False)
+    ] + args
+    result = self.m.python(
+      name, self.resource('fileutil.py'), args=args,
+      timeout=timeout, step_test_data=step_test_data, stdout=stdout,
+      infra_step=True)
+    j = result.json.outputs['main']

[dnj, 2017/05/31 16:50:40]

Only one output, WDYT about naming default?

[iannucci, 2017/05/31 22:59:26]

at first I thought that other things would need their own json, but nothing else
actually does. Done.

+    if not j['ok']:
+      # pylint thinks this isn't a standard exception... silly pylint.
+      # pylint: disable=nonstandard-exception
+      raise self.Error(name, j['errno_name'], j['message'])
+    return result
+
+  def copy(self, name, source, dest, timeout=None):
+    """Copies a file (including mode bits) from source to destination on the
+    local filesystem.
+
+    Behaves identically to shutil.copy.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The path to the file you want to copy.
+      dest (Path) - The path to the destination file name. If this path exists
+        and is a directory, the basename of `source` will be appended to derive
+        a path to a destination file.
+      timeout (int) - Maximum number of seconds to allow this command to run.
+
+    Raises file.Error
+    """
+    self.m.path.assert_absolute(source)
+    self.m.path.assert_absolute(dest)
+    self._run(name, ['copy', source, dest], timeout)
+    self.m.path.mock_copy_paths(source, dest)
+
+  def copytree(self, name, source, dest, symlinks=False, timeout=None):
+    """Recursively copies a directory tree.
+
+    Behaves identically to shutil.copytree.
+    `dest` must not exist.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The path of the directory to copy.
+      dest (Path) - The place where you want the recursive copy to show up. This
+        must not already exist.
+      timeout (int) - Maximum number of seconds to allow this command to run.

[dnj, 2017/05/31 16:50:40]

nit: Document "symlinks"

[iannucci, 2017/05/31 22:59:26]

Done.

+
+    Raises file.Error
+    """
+    self.m.path.assert_absolute(source)
+    self.m.path.assert_absolute(dest)
+    prefix = ['--symlinks'] if symlinks else []
+    self._run(name, prefix+['copytree', source, dest], timeout)
+    self.m.path.mock_copy_paths(source, dest)
+
+  def move(self, name, source, dest):
+    """Moves a file or directory.
+
+    Behaves identically to shutil.move.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The path of the item to move.
+      dest (Path) - The new name of the item.
+
+    Raises file.Error
+    """
+    self.m.path.assert_absolute(source)
+    self.m.path.assert_absolute(dest)
+    self._run(name, ['move', source, dest], timeout=5)

[dnj, 2017/05/31 16:50:40]

This timeout might actually be inadequate if it's cross-filesystem moving large
file?

[iannucci, 2017/05/31 22:59:26]

blergh, true. Removed.

+    self.m.path.mock_copy_paths(source, dest)
+    self.m.path.mock_remove_paths(source)
+
+  def read_text(self, name, source, test_data=''):
+    """Reads a file as UTF-8 encoded text.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The path of the file to read.
+      test_data (str) - Some default data for this step to return when running
+        under simulation.
+
+    Returns (str) - The content of the file.
+
+    Raises file.Error
+    """
+    self.m.path.assert_absolute(source)
+    step_test_data = lambda: self.test_api.read_text(test_data)
+    result = self._run(name, ['copy', source, self.m.raw_io.output_text()],
+                       timeout=30, step_test_data=step_test_data)
+    return result.raw_io.output_text
+
+  def write_text(self, name, dest, text_data):
+    """Write the given UTF-8 encoded `text_data` to `dest`.
+
+    Args:
+      name (str) - The name of the step.
+      dest (Path) - The path of the file to write.
+      text_data (str) - The UTF-8 encoded data to write.
+
+    Raises file.Error.
+    """
+    self.m.path.assert_absolute(dest)
+    self._run(name, ['copy', self.m.raw_io.input_text(text_data), dest],
+              timeout=30)
+    self.m.path.mock_add_paths(dest)
+
+  def glob_paths(self, name, pattern, test_data=(), timeout=None):
+    """Performs glob expansion on `pattern`.
+
+    glob rules for `pattern` follow the same syntax as for the python `glob`
+    stdlib module.
+
+    Args:
+      name (str) - The name of the step.
+      pattern (str|Path) - The glob pattern to expand.
+      test_data (iterable[str|Path]) - Some default paths for this step to
+        return when running under simulation.
+      timeout (int) - Maximum number of seconds to allow this command to run.
+
+    Returns list[Path] - All paths found.
+
+    Raises file.Error.
+    """
+    ret = self._run(
+      name, ['glob', pattern], timeout,
+      lambda: self.test_api.glob_paths(test_data),
+      self.m.raw_io.output_text())
+    return map(self.m.path.abs_to_path, ret.stdout.splitlines())
+
+  def remove(self, name, source):
+    """Remove a file.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The file to remove.
+
+    Raises file.Error.
+    """
+    self.m.path.assert_absolute(source)
+    self._run(name, ['remove', source], timeout=5)
+    self.m.path.mock_remove_paths(source)
+
+  def listdir(self, name, source, test_data=(), timeout=None):
+    """List all files inside a directory.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The directory to list.
+      test_data (iterable[str]) - Some default data for this step to return when
+        running under simulation.
+      timeout (int) - Maximum number of seconds to allow this command to run.

[dnj, 2017/05/31 16:50:40]

Is timeout useful here? I can't think of a meaningful way to set it. Maybe
numeric limit instead?

+
+    Returns list[str] - The names of the entries in the source directory. Note

[dnj, 2017/05/31 16:50:40]

QQ: why not Path objects?

[iannucci, 2017/05/31 22:59:26]

Because the usage of listdir frequently deals with just the filenames. We can
revisit this later if stuff tries to Pathify it (I'll see when I convert stuff
to use this)

+      that this is just the names of the files/folders, NOT Path objects. If you
+      want Path objects, join these with `source`, and use the path module's
+      `abs_to_path()` method.
+
+    Raises file.Error.
+    """
+    self.m.path.assert_absolute(source)
+    ret = self._run(
+      name, ['listdir', source], timeout,
+      lambda: self.test_api.listdir(test_data),
+      self.m.raw_io.output_text())
+    return ret.stdout.splitlines()
+
+  def ensure_directory(self, name, dest, mode=0777):
+    """Ensures that `dest` exists and is a directory.
+
+    Args:
+      name (str) - The name of the step.
+      dest (Path) - The directory to ensure.
+      mode (int) - The mode to use if the directory doesn't exist. This method
+        does not ensure the mode if the directory already exists (if you need
+        that behaviour, file a bug).
+
+    Raises file.Error if the path exists but is not a directory.
+    """
+    self.m.path.assert_absolute(dest)
+    self._run(
+      name, ['ensure-directory', '--mode', oct(mode), dest], timeout=5)
+    self.m.path.mock_add_paths(dest)
+
+  def rmtree(self, name, source, timeout=None):
+    """Recursively removes a directory.
+
+    This uses a native python on Linux/Mac, and uses `rd` on Windows to avoid
+    issues w.r.t. path lengths and read-only attributes. If the directory is
+    gone already, this returns without error.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The directory to remove.
+      timeout (int) - Maximum number of seconds to allow this command to run.
+
+    Raises file.Error.
+    """
+    self.m.path.assert_absolute(source)
+    self._run(name, ['rmtree', source], timeout)
+    self.m.path.mock_remove_paths(str(source))
+
+  def rmcontents(self, name, source, timeout=None):
+    """Similar to rmtree, but removes only contents not the directory.
+
+    This is useful e.g. when removing contents of current working directory.
+    Deleting current working directory makes all further getcwd calls fail
+    until chdir is called. chdir would be tricky in recipes, so we provide
+    a call that doesn't delete the directory itself.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The directory whose contents should be removed.
+      timeout (int) - Maximum number of seconds to allow this command to run.
+
+    Raises file.Error.
+    """
+    self.m.path.assert_absolute(source)
+    self._run(name, ['rmcontents', source], timeout)
+    self.m.path.mock_remove_paths(str(source)+self.m.path.sep)
+
+  def rmglob(self, name, source, pattern, timeout=None):
+    """Removes all entries in `source` matching the glob `pattern`.
+
+    Args:
+      name (str) - The name of the step.
+      source (Path) - The directory whose contents should be filtered and
+        removed.
+      pattern (str) - The glob pattern to apply under `source`. Anything
+        matching this pattern will be removed.
+      timeout (int) - Maximum number of seconds to allow this command to run.
+
+    Raises file.Error.
+    """
+    self.m.path.assert_absolute(source)
+    self._run(name, ['rmglob', source, pattern], timeout)
+
+    src = str(source)
+    def filt(p):
+      assert p.startswith(src), (src, p)
+      return fnmatch.fnmatch(p[len(src)+1:].split(os.path.sep)[0], pattern)
+    self.m.path.mock_remove_paths(str(source), filt)