Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(123)

Unified Diff: tools/telemetry/third_party/gsutil/third_party/boto/boto/services/result.py

Issue 1260493004: Revert "Add gsutil 4.13 to telemetry/third_party" (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: tools/telemetry/third_party/gsutil/third_party/boto/boto/services/result.py
diff --git a/tools/telemetry/third_party/gsutil/third_party/boto/boto/services/result.py b/tools/telemetry/third_party/gsutil/third_party/boto/boto/services/result.py
deleted file mode 100644
index 879934323ba2bf8167eebd07c54bb42b3e5950ab..0000000000000000000000000000000000000000
--- a/tools/telemetry/third_party/gsutil/third_party/boto/boto/services/result.py
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-import os
-from datetime import datetime, timedelta
-from boto.utils import parse_ts
-import boto
-
-class ResultProcessor(object):
-
- LogFileName = 'log.csv'
-
- def __init__(self, batch_name, sd, mimetype_files=None):
- self.sd = sd
- self.batch = batch_name
- self.log_fp = None
- self.num_files = 0
- self.total_time = 0
- self.min_time = timedelta.max
- self.max_time = timedelta.min
- self.earliest_time = datetime.max
- self.latest_time = datetime.min
- self.queue = self.sd.get_obj('output_queue')
- self.domain = self.sd.get_obj('output_domain')
-
- def calculate_stats(self, msg):
- start_time = parse_ts(msg['Service-Read'])
- end_time = parse_ts(msg['Service-Write'])
- elapsed_time = end_time - start_time
- if elapsed_time > self.max_time:
- self.max_time = elapsed_time
- if elapsed_time < self.min_time:
- self.min_time = elapsed_time
- self.total_time += elapsed_time.seconds
- if start_time < self.earliest_time:
- self.earliest_time = start_time
- if end_time > self.latest_time:
- self.latest_time = end_time
-
- def log_message(self, msg, path):
- keys = sorted(msg.keys())
- if not self.log_fp:
- self.log_fp = open(os.path.join(path, self.LogFileName), 'a')
- line = ','.join(keys)
- self.log_fp.write(line+'\n')
- values = []
- for key in keys:
- value = msg[key]
- if value.find(',') > 0:
- value = '"%s"' % value
- values.append(value)
- line = ','.join(values)
- self.log_fp.write(line+'\n')
-
- def process_record(self, record, path, get_file=True):
- self.log_message(record, path)
- self.calculate_stats(record)
- outputs = record['OutputKey'].split(',')
- if 'OutputBucket' in record:
- bucket = boto.lookup('s3', record['OutputBucket'])
- else:
- bucket = boto.lookup('s3', record['Bucket'])
- for output in outputs:
- if get_file:
- key_name = output.split(';')[0]
- key = bucket.lookup(key_name)
- file_name = os.path.join(path, key_name)
- print('retrieving file: %s to %s' % (key_name, file_name))
- key.get_contents_to_filename(file_name)
- self.num_files += 1
-
- def get_results_from_queue(self, path, get_file=True, delete_msg=True):
- m = self.queue.read()
- while m:
- if 'Batch' in m and m['Batch'] == self.batch:
- self.process_record(m, path, get_file)
- if delete_msg:
- self.queue.delete_message(m)
- m = self.queue.read()
-
- def get_results_from_domain(self, path, get_file=True):
- rs = self.domain.query("['Batch'='%s']" % self.batch)
- for item in rs:
- self.process_record(item, path, get_file)
-
- def get_results_from_bucket(self, path):
- bucket = self.sd.get_obj('output_bucket')
- if bucket:
- print('No output queue or domain, just retrieving files from output_bucket')
- for key in bucket:
- file_name = os.path.join(path, key)
- print('retrieving file: %s to %s' % (key, file_name))
- key.get_contents_to_filename(file_name)
- self.num_files + 1
-
- def get_results(self, path, get_file=True, delete_msg=True):
- if not os.path.isdir(path):
- os.mkdir(path)
- if self.queue:
- self.get_results_from_queue(path, get_file)
- elif self.domain:
- self.get_results_from_domain(path, get_file)
- else:
- self.get_results_from_bucket(path)
- if self.log_fp:
- self.log_fp.close()
- print('%d results successfully retrieved.' % self.num_files)
- if self.num_files > 0:
- self.avg_time = float(self.total_time)/self.num_files
- print('Minimum Processing Time: %d' % self.min_time.seconds)
- print('Maximum Processing Time: %d' % self.max_time.seconds)
- print('Average Processing Time: %f' % self.avg_time)
- self.elapsed_time = self.latest_time-self.earliest_time
- print('Elapsed Time: %d' % self.elapsed_time.seconds)
- tput = 1.0 / ((self.elapsed_time.seconds/60.0) / self.num_files)
- print('Throughput: %f transactions / minute' % tput)
-

Powered by Google App Engine
This is Rietveld 408576698