luci-py: Tools for working with BSD style ar archives.

client/libs/ar/cli.py

+# Copyright 2016 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+from __future__ import print_function
+
+import argparse
+import os
+import stat
+import sys
+import time
+
+from .writer import ArDefaultWriter
+from .reader import ArDefaultReader
+
+
+class Timing(object):
+  def __init__(self, every):
+    self.every = every
+    self.start = time.time()
+    self.filecount = 0
+    self.lastreport = 0
+
+  def inc(self):
+    self.filecount += 1
+    if (self.filecount - self.lastreport) >= self.every:
+      self.report()
+
+  def report(self):
+    if self.every:
+      t = time.time()-self.start
+      print("Took %f for %i files == %f files/second" % (

[M-A Ruel, 2016/06/09 21:50:43]

use single quotes

[mithro, 2016/06/14 12:15:55]

Done.

+        t, self.filecount, self.filecount/t), file=sys.stderr)
+    self.lastreport = self.filecount
+
+  def __del__(self):

[M-A Ruel, 2016/06/09 21:50:43]

Use __exit__ instead, don't overide __del__

we already have a class like that; utils.Profiler

[mithro, 2016/06/14 12:15:55]

They are doing slightly different things, the Timing class exists so you get
frequent "files per second" reports rather than trying to record proper
statistics.

I renamed it to be "ProgressReporter" to hopefully be clearer.

+    self.report()
+
+
+def create_cmd(filename, dirs, read_ahead=0, timing=False, verbose=True):
+  arfile = ArDefaultWriter(filename)
+  timing = Timing(timing)

[M-A Ruel, 2016/06/09 21:50:43]

e.g.:
with Timing() as t:

[mithro, 2016/06/14 12:15:55]

Obsolete?

+  for path in dirs:
+    for dirpath, child_dirs, filenames in os.walk(path):
+      # In-place sort the child_dirs so we walk in lexicographical order
+      child_dirs.sort()
+
+      for fn in sorted(filenames):
+        fp = os.path.join(dirpath, fn)
+
+        if verbose:
+          print(fp, file=sys.stderr)

[M-A Ruel, 2016/06/09 21:50:43]

logging.info()

[mithro, 2016/06/14 12:15:55]

Logging adds a bunch of extra stuff. We want just the file names listed here.

+
+        with open(fp, 'r') as f:

[M-A Ruel, 2016/06/09 21:50:43]

'rb' is important

[mithro, 2016/06/14 12:15:55]

Done.

+          # If a file is small, it is cheaper to just read the file rather than
+          # doing a stat
+          data = f.read(read_ahead)
+          if len(data) < read_ahead:
+            arfile.add(fp, data)
+          else:
+            size = os.stat(fp).st_size
+            arfile.header(fp, size)
+            arfile.write(data)
+            arfile.write(f)
+
+        timing.inc()
+
+  arfile.close()
+
+
+def list_cmd(filename, timing=False, check=False):
+  arfile = ArDefaultReader(filename, check)
+  timing = Timing(timing)
+  for fp, _, _ in arfile:
+    print(fp)
+    timing.inc()
+
+
+def extract_cmd(
+      filename, timing=False, verbose=False, blocksize=1024*64, check=False):
+  arfile = ArDefaultReader(filename, check)
+  timing = Timing(timing)
+  for fp, size, ifd in arfile:
+    if verbose:
+      print(fp, file=sys.stderr)
+
+    start = ifd.tell()
+    try:
+      os.makedirs(os.path.dirname(fp))
+    except OSError:
+      pass
+
+    with open(fp, 'w') as ofd:
+      written = 0
+      while written < size:
+        readsize = min(blocksize, size-written)
+        ofd.write(ifd.read(readsize))
+
+    end = ifd.tell()
+    assert end-start == size
+
+
+def main(name, args):
+  parser = argparse.ArgumentParser(
+    prog=name,
+    description='Command line tool for creating and extracting ar files.')

[M-A Ruel, 2016/06/09 21:50:43]

One thing I like is to put this as the module docstring then use here:

sys.modules[__name__].__doc__

[mithro, 2016/06/14 12:15:55]

Done.

+  subparsers = parser.add_subparsers(
+    dest='mode', help='sub-command help')
+
+  # Create command
+  parser_create = subparsers.add_parser(
+    'create', help='Create a new ar file')
+  parser_create.add_argument(
+    "-v", "--verbose",
+    action="store_true",
+    help="Print file names to stderr while creating the archive")
+  parser_create.add_argument(
+    "-r", "--read-ahead",
+    type=int, default=1024*64,
+    help="Amount of data to read-ahead before doing a stat.")
+  parser_create.add_argument(
+    "-f", "--filename",
+    type=argparse.FileType('w'), default=sys.stdout,

[M-A Ruel, 2016/06/09 21:50:43]

'wb' and 'rb' below

[mithro, 2016/06/14 12:15:55]

Done.

+    help="ar file to use")
+  parser_create.add_argument(
+    'dirs', nargs='+', help='Directory or file to add to the ar file')
+
+  # List command
+  parser_list = subparsers.add_parser('list', help='List a new ar file')
+
+  # Extract command
+  parser_extract = subparsers.add_parser(
+    'extract', help='Extract an existing ar file to current directory')
+  parser_extract.add_argument(
+    "-v", "--verbose",
+    action="store_true",
+    help='Output file names as extracted.')
+  parser_extract.add_argument(
+    "-f", "--file",
+    type=argparse.FileType('r'), default=sys.stdin,
+    help="ar file to use")
+
+  # Add to output commands
+  for p in parser_list, parser_extract:
+    p.add_argument(
+      "-f", "--filename",
+      type=argparse.FileType('r'), default=sys.stdin,
+      help="ar file to use")
+    p.add_argument(
+      "--check",
+      action="store_true",
+      help='Check the ar was created by ArDefaultWriter')
+
+  # Add to all commands
+  for p in parser_create, parser_list, parser_extract:
+    p.add_argument(
+      "-t", "--timing",
+      type=int, default=0,
+      help='Output timing information every N files.')
+
+  args = parser.parse_args(args)
+  mode = eval("%s_cmd" % args.mode)

[M-A Ruel, 2016/06/09 21:50:43]

mode = getattr(sys.modules[__name__], args.mode + '_cmd')

[mithro, 2016/06/14 12:15:55]

Done.

+  del args.mode
+  mode(**args.__dict__)
+
+
+if __name__ == "__main__":
+  main("artool", sys.argv[1:])