| Index: tools/telemetry/third_party/gsutilz/gslib/progress_callback.py
|
| diff --git a/tools/telemetry/third_party/gsutilz/gslib/progress_callback.py b/tools/telemetry/third_party/gsutilz/gslib/progress_callback.py
|
| deleted file mode 100644
|
| index 73ee490be83c484cb6a1cabb935a32a1ed82c8f4..0000000000000000000000000000000000000000
|
| --- a/tools/telemetry/third_party/gsutilz/gslib/progress_callback.py
|
| +++ /dev/null
|
| @@ -1,162 +0,0 @@
|
| -# -*- coding: utf-8 -*-
|
| -# Copyright 2014 Google Inc. All Rights Reserved.
|
| -#
|
| -# Licensed under the Apache License, Version 2.0 (the "License");
|
| -# you may not use this file except in compliance with the License.
|
| -# You may obtain a copy of the License at
|
| -#
|
| -# http://www.apache.org/licenses/LICENSE-2.0
|
| -#
|
| -# Unless required by applicable law or agreed to in writing, software
|
| -# distributed under the License is distributed on an "AS IS" BASIS,
|
| -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| -# See the License for the specific language governing permissions and
|
| -# limitations under the License.
|
| -"""Helper functions for progress callbacks."""
|
| -
|
| -import logging
|
| -import sys
|
| -
|
| -from gslib.util import MakeHumanReadable
|
| -from gslib.util import UTF8
|
| -
|
| -# Default upper and lower bounds for progress callback frequency.
|
| -_START_BYTES_PER_CALLBACK = 1024*64
|
| -_MAX_BYTES_PER_CALLBACK = 1024*1024*100
|
| -
|
| -# Max width of URL to display in progress indicator. Wide enough to allow
|
| -# 15 chars for x/y display on an 80 char wide terminal.
|
| -MAX_PROGRESS_INDICATOR_COLUMNS = 65
|
| -
|
| -
|
| -class ProgressCallbackWithBackoff(object):
|
| - """Makes progress callbacks with exponential backoff to a maximum value.
|
| -
|
| - This prevents excessive log message output.
|
| - """
|
| -
|
| - def __init__(self, total_size, callback_func,
|
| - start_bytes_per_callback=_START_BYTES_PER_CALLBACK,
|
| - max_bytes_per_callback=_MAX_BYTES_PER_CALLBACK,
|
| - calls_per_exponent=10):
|
| - """Initializes the callback with backoff.
|
| -
|
| - Args:
|
| - total_size: Total bytes to process. If this is None, size is not known
|
| - at the outset.
|
| - callback_func: Func of (int: processed_so_far, int: total_bytes)
|
| - used to make callbacks.
|
| - start_bytes_per_callback: Lower bound of bytes per callback.
|
| - max_bytes_per_callback: Upper bound of bytes per callback.
|
| - calls_per_exponent: Number of calls to make before reducing rate.
|
| - """
|
| - self._bytes_per_callback = start_bytes_per_callback
|
| - self._callback_func = callback_func
|
| - self._calls_per_exponent = calls_per_exponent
|
| - self._max_bytes_per_callback = max_bytes_per_callback
|
| - self._total_size = total_size
|
| -
|
| - self._bytes_processed_since_callback = 0
|
| - self._callbacks_made = 0
|
| - self._total_bytes_processed = 0
|
| -
|
| - def Progress(self, bytes_processed):
|
| - """Tracks byte processing progress, making a callback if necessary."""
|
| - self._bytes_processed_since_callback += bytes_processed
|
| - if (self._bytes_processed_since_callback > self._bytes_per_callback or
|
| - (self._total_bytes_processed + self._bytes_processed_since_callback >=
|
| - self._total_size and self._total_size is not None)):
|
| - self._total_bytes_processed += self._bytes_processed_since_callback
|
| - # TODO: We check if >= total_size and truncate because JSON uploads count
|
| - # headers+metadata during their send progress. If the size is unknown,
|
| - # we can't do this and the progress message will make it appear that we
|
| - # send more than the original stream.
|
| - if self._total_size is not None:
|
| - bytes_sent = min(self._total_bytes_processed, self._total_size)
|
| - else:
|
| - bytes_sent = self._total_bytes_processed
|
| - self._callback_func(bytes_sent, self._total_size)
|
| - self._bytes_processed_since_callback = 0
|
| - self._callbacks_made += 1
|
| -
|
| - if self._callbacks_made > self._calls_per_exponent:
|
| - self._bytes_per_callback = min(self._bytes_per_callback * 2,
|
| - self._max_bytes_per_callback)
|
| - self._callbacks_made = 0
|
| -
|
| -
|
| -def ConstructAnnounceText(operation_name, url_string):
|
| - """Constructs announce text for ongoing operations on url_to_display.
|
| -
|
| - This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS.
|
| - Thus, concurrent output (gsutil -m) leaves progress counters in a readable
|
| - (fixed) position.
|
| -
|
| - Args:
|
| - operation_name: String describing the operation, i.e.
|
| - 'Uploading' or 'Hashing'.
|
| - url_string: String describing the file/object being processed.
|
| -
|
| - Returns:
|
| - Formatted announce text for outputting operation progress.
|
| - """
|
| - # Operation name occupies 11 characters (enough for 'Downloading'), plus a
|
| - # space. The rest is used for url_to_display. If a longer operation name is
|
| - # used, it will be truncated. We can revisit this size if we need to support
|
| - # a longer operation, but want to make sure the terminal output is meaningful.
|
| - justified_op_string = operation_name[:11].ljust(12)
|
| - start_len = len(justified_op_string)
|
| - end_len = len(': ')
|
| - if (start_len + len(url_string) + end_len >
|
| - MAX_PROGRESS_INDICATOR_COLUMNS):
|
| - ellipsis_len = len('...')
|
| - url_string = '...%s' % url_string[
|
| - -(MAX_PROGRESS_INDICATOR_COLUMNS - start_len - end_len - ellipsis_len):]
|
| - base_announce_text = '%s%s:' % (justified_op_string, url_string)
|
| - format_str = '{0:%ds}' % MAX_PROGRESS_INDICATOR_COLUMNS
|
| - return format_str.format(base_announce_text.encode(UTF8))
|
| -
|
| -
|
| -class FileProgressCallbackHandler(object):
|
| - """Outputs progress info for large operations like file copy or hash."""
|
| -
|
| - def __init__(self, announce_text, logger):
|
| - """Initializes the callback handler.
|
| -
|
| - Args:
|
| - announce_text: String describing the operation.
|
| - logger: For outputting log messages.
|
| - """
|
| - self._announce_text = announce_text
|
| - self._logger = logger
|
| - # Ensures final newline is written once even if we get multiple callbacks.
|
| - self._last_byte_written = False
|
| -
|
| - # Function signature is in boto callback format, which cannot be changed.
|
| - def call(self, # pylint: disable=invalid-name
|
| - total_bytes_processed,
|
| - total_size):
|
| - """Prints an overwriting line to stderr describing the operation progress.
|
| -
|
| - Args:
|
| - total_bytes_processed: Number of bytes processed so far.
|
| - total_size: Total size of the ongoing operation.
|
| - """
|
| - if not self._logger.isEnabledFor(logging.INFO) or self._last_byte_written:
|
| - return
|
| -
|
| - # Handle streaming case specially where we don't know the total size:
|
| - if total_size:
|
| - total_size_string = '/%s' % MakeHumanReadable(total_size)
|
| - else:
|
| - total_size_string = ''
|
| - # Use sys.stderr.write instead of self.logger.info so progress messages
|
| - # output on a single continuously overwriting line.
|
| - # TODO: Make this work with logging.Logger.
|
| - sys.stderr.write('%s%s%s \r' % (
|
| - self._announce_text,
|
| - MakeHumanReadable(total_bytes_processed),
|
| - total_size_string))
|
| - if total_size and total_bytes_processed == total_size:
|
| - self._last_byte_written = True
|
| - sys.stderr.write('\n')
|
|
|