Index: tools/telemetry/third_party/gsutil/gslib/progress_callback.py |
diff --git a/tools/telemetry/third_party/gsutil/gslib/progress_callback.py b/tools/telemetry/third_party/gsutil/gslib/progress_callback.py |
deleted file mode 100644 |
index 73ee490be83c484cb6a1cabb935a32a1ed82c8f4..0000000000000000000000000000000000000000 |
--- a/tools/telemetry/third_party/gsutil/gslib/progress_callback.py |
+++ /dev/null |
@@ -1,162 +0,0 @@ |
-# -*- coding: utf-8 -*- |
-# Copyright 2014 Google Inc. All Rights Reserved. |
-# |
-# Licensed under the Apache License, Version 2.0 (the "License"); |
-# you may not use this file except in compliance with the License. |
-# You may obtain a copy of the License at |
-# |
-# http://www.apache.org/licenses/LICENSE-2.0 |
-# |
-# Unless required by applicable law or agreed to in writing, software |
-# distributed under the License is distributed on an "AS IS" BASIS, |
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
-# See the License for the specific language governing permissions and |
-# limitations under the License. |
-"""Helper functions for progress callbacks.""" |
- |
-import logging |
-import sys |
- |
-from gslib.util import MakeHumanReadable |
-from gslib.util import UTF8 |
- |
-# Default upper and lower bounds for progress callback frequency. |
-_START_BYTES_PER_CALLBACK = 1024*64 |
-_MAX_BYTES_PER_CALLBACK = 1024*1024*100 |
- |
-# Max width of URL to display in progress indicator. Wide enough to allow |
-# 15 chars for x/y display on an 80 char wide terminal. |
-MAX_PROGRESS_INDICATOR_COLUMNS = 65 |
- |
- |
-class ProgressCallbackWithBackoff(object): |
- """Makes progress callbacks with exponential backoff to a maximum value. |
- |
- This prevents excessive log message output. |
- """ |
- |
- def __init__(self, total_size, callback_func, |
- start_bytes_per_callback=_START_BYTES_PER_CALLBACK, |
- max_bytes_per_callback=_MAX_BYTES_PER_CALLBACK, |
- calls_per_exponent=10): |
- """Initializes the callback with backoff. |
- |
- Args: |
- total_size: Total bytes to process. If this is None, size is not known |
- at the outset. |
- callback_func: Func of (int: processed_so_far, int: total_bytes) |
- used to make callbacks. |
- start_bytes_per_callback: Lower bound of bytes per callback. |
- max_bytes_per_callback: Upper bound of bytes per callback. |
- calls_per_exponent: Number of calls to make before reducing rate. |
- """ |
- self._bytes_per_callback = start_bytes_per_callback |
- self._callback_func = callback_func |
- self._calls_per_exponent = calls_per_exponent |
- self._max_bytes_per_callback = max_bytes_per_callback |
- self._total_size = total_size |
- |
- self._bytes_processed_since_callback = 0 |
- self._callbacks_made = 0 |
- self._total_bytes_processed = 0 |
- |
- def Progress(self, bytes_processed): |
- """Tracks byte processing progress, making a callback if necessary.""" |
- self._bytes_processed_since_callback += bytes_processed |
- if (self._bytes_processed_since_callback > self._bytes_per_callback or |
- (self._total_bytes_processed + self._bytes_processed_since_callback >= |
- self._total_size and self._total_size is not None)): |
- self._total_bytes_processed += self._bytes_processed_since_callback |
- # TODO: We check if >= total_size and truncate because JSON uploads count |
- # headers+metadata during their send progress. If the size is unknown, |
- # we can't do this and the progress message will make it appear that we |
- # send more than the original stream. |
- if self._total_size is not None: |
- bytes_sent = min(self._total_bytes_processed, self._total_size) |
- else: |
- bytes_sent = self._total_bytes_processed |
- self._callback_func(bytes_sent, self._total_size) |
- self._bytes_processed_since_callback = 0 |
- self._callbacks_made += 1 |
- |
- if self._callbacks_made > self._calls_per_exponent: |
- self._bytes_per_callback = min(self._bytes_per_callback * 2, |
- self._max_bytes_per_callback) |
- self._callbacks_made = 0 |
- |
- |
-def ConstructAnnounceText(operation_name, url_string): |
- """Constructs announce text for ongoing operations on url_to_display. |
- |
- This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS. |
- Thus, concurrent output (gsutil -m) leaves progress counters in a readable |
- (fixed) position. |
- |
- Args: |
- operation_name: String describing the operation, i.e. |
- 'Uploading' or 'Hashing'. |
- url_string: String describing the file/object being processed. |
- |
- Returns: |
- Formatted announce text for outputting operation progress. |
- """ |
- # Operation name occupies 11 characters (enough for 'Downloading'), plus a |
- # space. The rest is used for url_to_display. If a longer operation name is |
- # used, it will be truncated. We can revisit this size if we need to support |
- # a longer operation, but want to make sure the terminal output is meaningful. |
- justified_op_string = operation_name[:11].ljust(12) |
- start_len = len(justified_op_string) |
- end_len = len(': ') |
- if (start_len + len(url_string) + end_len > |
- MAX_PROGRESS_INDICATOR_COLUMNS): |
- ellipsis_len = len('...') |
- url_string = '...%s' % url_string[ |
- -(MAX_PROGRESS_INDICATOR_COLUMNS - start_len - end_len - ellipsis_len):] |
- base_announce_text = '%s%s:' % (justified_op_string, url_string) |
- format_str = '{0:%ds}' % MAX_PROGRESS_INDICATOR_COLUMNS |
- return format_str.format(base_announce_text.encode(UTF8)) |
- |
- |
-class FileProgressCallbackHandler(object): |
- """Outputs progress info for large operations like file copy or hash.""" |
- |
- def __init__(self, announce_text, logger): |
- """Initializes the callback handler. |
- |
- Args: |
- announce_text: String describing the operation. |
- logger: For outputting log messages. |
- """ |
- self._announce_text = announce_text |
- self._logger = logger |
- # Ensures final newline is written once even if we get multiple callbacks. |
- self._last_byte_written = False |
- |
- # Function signature is in boto callback format, which cannot be changed. |
- def call(self, # pylint: disable=invalid-name |
- total_bytes_processed, |
- total_size): |
- """Prints an overwriting line to stderr describing the operation progress. |
- |
- Args: |
- total_bytes_processed: Number of bytes processed so far. |
- total_size: Total size of the ongoing operation. |
- """ |
- if not self._logger.isEnabledFor(logging.INFO) or self._last_byte_written: |
- return |
- |
- # Handle streaming case specially where we don't know the total size: |
- if total_size: |
- total_size_string = '/%s' % MakeHumanReadable(total_size) |
- else: |
- total_size_string = '' |
- # Use sys.stderr.write instead of self.logger.info so progress messages |
- # output on a single continuously overwriting line. |
- # TODO: Make this work with logging.Logger. |
- sys.stderr.write('%s%s%s \r' % ( |
- self._announce_text, |
- MakeHumanReadable(total_bytes_processed), |
- total_size_string)) |
- if total_size and total_bytes_processed == total_size: |
- self._last_byte_written = True |
- sys.stderr.write('\n') |