Index: tools/telemetry/third_party/gsutil/third_party/boto/bin/dynamodb_load |
diff --git a/tools/telemetry/third_party/gsutil/third_party/boto/bin/dynamodb_load b/tools/telemetry/third_party/gsutil/third_party/boto/bin/dynamodb_load |
deleted file mode 100644 |
index 46a8d392ff8b58b8a56f5e158815216ac6970b3d..0000000000000000000000000000000000000000 |
--- a/tools/telemetry/third_party/gsutil/third_party/boto/bin/dynamodb_load |
+++ /dev/null |
@@ -1,109 +0,0 @@ |
-#!/usr/bin/env python |
- |
-import argparse |
-import os |
- |
-import boto |
-from boto.compat import json |
-from boto.dynamodb.schema import Schema |
- |
- |
-DESCRIPTION = """Load data into one or more DynamoDB tables. |
- |
-For each table, data is read from two files: |
- - {table_name}.metadata for the table's name, schema and provisioned |
- throughput (only required if creating the table). |
- - {table_name}.data for the table's actual contents. |
- |
-Both files are searched for in the current directory. To read them from |
-somewhere else, use the --in-dir parameter. |
- |
-This program does not wipe the tables prior to loading data. However, any |
-items present in the data files will overwrite the table's contents. |
-""" |
- |
- |
-def _json_iterload(fd): |
- """Lazily load newline-separated JSON objects from a file-like object.""" |
- buffer = "" |
- eof = False |
- while not eof: |
- try: |
- # Add a line to the buffer |
- buffer += fd.next() |
- except StopIteration: |
- # We can't let that exception bubble up, otherwise the last |
- # object in the file will never be decoded. |
- eof = True |
- try: |
- # Try to decode a JSON object. |
- json_object = json.loads(buffer.strip()) |
- |
- # Success: clear the buffer (everything was decoded). |
- buffer = "" |
- except ValueError: |
- if eof and buffer.strip(): |
- # No more lines to load and the buffer contains something other |
- # than whitespace: the file is, in fact, malformed. |
- raise |
- # We couldn't decode a complete JSON object: load more lines. |
- continue |
- |
- yield json_object |
- |
- |
-def create_table(metadata_fd): |
- """Create a table from a metadata file-like object.""" |
- |
- |
-def load_table(table, in_fd): |
- """Load items into a table from a file-like object.""" |
- for i in _json_iterload(in_fd): |
- # Convert lists back to sets. |
- data = {} |
- for k, v in i.iteritems(): |
- if isinstance(v, list): |
- data[k] = set(v) |
- else: |
- data[k] = v |
- table.new_item(attrs=data).put() |
- |
- |
-def dynamodb_load(tables, in_dir, create_tables): |
- conn = boto.connect_dynamodb() |
- for t in tables: |
- metadata_file = os.path.join(in_dir, "%s.metadata" % t) |
- data_file = os.path.join(in_dir, "%s.data" % t) |
- if create_tables: |
- with open(metadata_file) as meta_fd: |
- metadata = json.load(meta_fd) |
- table = conn.create_table( |
- name=t, |
- schema=Schema(metadata["schema"]), |
- read_units=metadata["read_units"], |
- write_units=metadata["write_units"], |
- ) |
- table.refresh(wait_for_active=True) |
- else: |
- table = conn.get_table(t) |
- |
- with open(data_file) as in_fd: |
- load_table(table, in_fd) |
- |
- |
-if __name__ == "__main__": |
- parser = argparse.ArgumentParser( |
- prog="dynamodb_load", |
- description=DESCRIPTION |
- ) |
- parser.add_argument( |
- "--create-tables", |
- action="store_true", |
- help="Create the tables if they don't exist already (without this flag, attempts to load data into non-existing tables fail)." |
- ) |
- parser.add_argument("--in-dir", default=".") |
- parser.add_argument("tables", metavar="TABLES", nargs="+") |
- |
- namespace = parser.parse_args() |
- |
- dynamodb_load(namespace.tables, namespace.in_dir, namespace.create_tables) |