luci-py: Tools for working with BSD style ar archives.

client/libs/ar/cli.py

+# Copyright 2016 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+"""Command line tool for creating and extracting ar files."""
+
+from __future__ import print_function
+
+import argparse
+import os
+import stat
+import sys
+import time
+
+from .writer import ArDefaultWriter
+from .reader import ArDefaultReader
+
+class ProgressReporter(object):
+  def __init__(self, every):
+    self.every = int(every)
+    self.start = time.time()
+    self.filecount = 0
+    self.lastreport = 0
+
+  def inc(self):
+    self.filecount += 1
+    if (self.filecount - self.lastreport) >= self.every:
+      self.report()
+
+  def report(self):
+    if self.every:
+      t = time.time()-self.start
+      print('Took %f for %i files == %f files/second' % (
+        t, self.filecount, self.filecount/t), file=sys.stderr)
+    self.lastreport = self.filecount
+
+  def __del__(self):
+    self.report()
+
+
+def create_cmd(filename, dirs, progress, read_ahead=0, verbose=True):

[M-A Ruel, 2016/06/14 13:26:16]

remove verbose flag.
do not put a default value to read_ahead.

+  arfile = ArDefaultWriter(filename)
+  for path in dirs:
+    for dirpath, child_dirs, filenames in os.walk(path):
+      # In-place sort the child_dirs so we walk in lexicographical order
+      child_dirs.sort()
+
+      for fn in sorted(filenames):
+        fp = os.path.join(dirpath, fn)
+
+        if verbose:
+          print(fp, file=sys.stderr)

[M-A Ruel, 2016/06/14 13:26:16]

Please use logging

+
+        with open(fp, 'rb') as f:
+          # If a file is small, it is cheaper to just read the file rather than
+          # doing a stat
+          data = f.read(read_ahead)
+          if len(data) < read_ahead:
+            arfile.add(fp, data)
+          else:
+            size = os.stat(fp).st_size
+            arfile.header(fp, size)
+            arfile.write(data)
+            arfile.write(f)
+
+        progress.inc()
+
+  arfile.close()
+
+
+def list_cmd(filename, progress, check=False):

[M-A Ruel, 2016/06/14 13:26:16]

remove default value

+  arfile = ArDefaultReader(filename, check)
+  for fp, _, _ in arfile:

[M-A Ruel, 2016/06/14 13:26:16]

for fp, _, _ in ArDefaultReader(filename, check):

+    print(fp)
+    progress.inc()
+
+
+def extract_cmd(
+      filename, progress, verbose=False, blocksize=1024*64, check=False):

[M-A Ruel, 2016/06/14 13:26:16]

remove the verbose flag, use logging
remove default values

+  arfile = ArDefaultReader(filename, check)
+  for fp, size, ifd in arfile:
+    if verbose:
+      print(fp, file=sys.stderr)
+
+    start = ifd.tell()
+    try:
+      os.makedirs(os.path.dirname(fp))

[M-A Ruel, 2016/06/14 13:26:16]

you should keep track of the directories that were created in a list and skip
these. Then directory creation failure can be a hard fail and you can call
os.mkdir(), which is likely (?) faster.

+    except OSError:
+      pass
+
+    with open(fp, 'wb') as ofd:
+      written = 0
+      while written < size:
+        readsize = min(blocksize, size-written)
+        ofd.write(ifd.read(readsize))
+
+    end = ifd.tell()
+    assert end-start == size
+
+    progress.inc()
+
+
+def main(name, args):
+  parser = argparse.ArgumentParser(
+    prog=name,
+    description=sys.modules[__name__].__doc__)
+  subparsers = parser.add_subparsers(
+    dest='mode', help='sub-command help')
+
+  # Create command
+  parser_create = subparsers.add_parser(
+    'create', help='Create a new ar file')
+  parser_create.add_argument(
+    "-r", "--read-ahead",
+    type=int, default=1024*64,
+    help="Amount of data to read-ahead before doing a stat.")
+  parser_create.add_argument(
+    "-f", "--filename",
+    type=argparse.FileType('wb'), default=sys.stdout,
+    help="ar file to use")
+  parser_create.add_argument(
+    'dirs', nargs='+', help='Directory or file to add to the ar file')
+
+  # List command
+  parser_list = subparsers.add_parser('list', help='List a new ar file')
+
+  # Extract command
+  parser_extract = subparsers.add_parser(
+    'extract', help='Extract an existing ar file to current directory')
+
+  # Add to output commands
+  for p in parser_list, parser_extract:
+    p.add_argument(
+      "-f", "--filename",
+      type=argparse.FileType('rb'), default=sys.stdin,
+      help="ar file to use")
+    p.add_argument(
+      "--check",
+      action="store_true",
+      help='Check the ar was created by ArDefaultWriter')
+
+  for p in parser_create, parser_extract:
+    p.add_argument(

[M-A Ruel, 2016/06/14 13:26:16]

add this flag as a global flag and call logging.basicConfig(level=) on it. We
have wrapping code too in utils/

+      "-v", "--verbose",
+      action="store_true",
+      help='Output file names to stderr while running.')
+
+  # Add to all commands
+  for p in parser_create, parser_list, parser_extract:
+    p.add_argument(
+      "-p", "--progress",
+      type=ProgressReporter, default='10000',
+      help='Output progress information every N files.')
+
+  args = parser.parse_args(args)
+  mode = getattr(sys.modules[__name__], args.mode + '_cmd')
+  del args.mode
+  mode(**args.__dict__)
+
+
+if __name__ == "__main__":
+  main("artool", sys.argv[1:])