| Index: tools/telemetry/third_party/gsutilz/third_party/boto/boto/sqs/queue.py
|
| diff --git a/tools/telemetry/third_party/gsutilz/third_party/boto/boto/sqs/queue.py b/tools/telemetry/third_party/gsutilz/third_party/boto/boto/sqs/queue.py
|
| deleted file mode 100644
|
| index bf3720d95f253271ee470305a46d943f784eb05c..0000000000000000000000000000000000000000
|
| --- a/tools/telemetry/third_party/gsutilz/third_party/boto/boto/sqs/queue.py
|
| +++ /dev/null
|
| @@ -1,492 +0,0 @@
|
| -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/
|
| -#
|
| -# Permission is hereby granted, free of charge, to any person obtaining a
|
| -# copy of this software and associated documentation files (the
|
| -# "Software"), to deal in the Software without restriction, including
|
| -# without limitation the rights to use, copy, modify, merge, publish, dis-
|
| -# tribute, sublicense, and/or sell copies of the Software, and to permit
|
| -# persons to whom the Software is furnished to do so, subject to the fol-
|
| -# lowing conditions:
|
| -#
|
| -# The above copyright notice and this permission notice shall be included
|
| -# in all copies or substantial portions of the Software.
|
| -#
|
| -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
| -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
| -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
| -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
| -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
| -# IN THE SOFTWARE.
|
| -
|
| -"""
|
| -Represents an SQS Queue
|
| -"""
|
| -from boto.compat import urllib
|
| -from boto.sqs.message import Message
|
| -
|
| -
|
| -class Queue(object):
|
| -
|
| - def __init__(self, connection=None, url=None, message_class=Message):
|
| - self.connection = connection
|
| - self.url = url
|
| - self.message_class = message_class
|
| - self.visibility_timeout = None
|
| -
|
| - def __repr__(self):
|
| - return 'Queue(%s)' % self.url
|
| -
|
| - def _id(self):
|
| - if self.url:
|
| - val = urllib.parse.urlparse(self.url)[2]
|
| - else:
|
| - val = self.url
|
| - return val
|
| - id = property(_id)
|
| -
|
| - def _name(self):
|
| - if self.url:
|
| - val = urllib.parse.urlparse(self.url)[2].split('/')[2]
|
| - else:
|
| - val = self.url
|
| - return val
|
| - name = property(_name)
|
| -
|
| - def _arn(self):
|
| - parts = self.id.split('/')
|
| - return 'arn:aws:sqs:%s:%s:%s' % (
|
| - self.connection.region.name, parts[1], parts[2])
|
| - arn = property(_arn)
|
| -
|
| - def startElement(self, name, attrs, connection):
|
| - return None
|
| -
|
| - def endElement(self, name, value, connection):
|
| - if name == 'QueueUrl':
|
| - self.url = value
|
| - elif name == 'VisibilityTimeout':
|
| - self.visibility_timeout = int(value)
|
| - else:
|
| - setattr(self, name, value)
|
| -
|
| - def set_message_class(self, message_class):
|
| - """
|
| - Set the message class that should be used when instantiating
|
| - messages read from the queue. By default, the class
|
| - :class:`boto.sqs.message.Message` is used but this can be overriden
|
| - with any class that behaves like a message.
|
| -
|
| - :type message_class: Message-like class
|
| - :param message_class: The new Message class
|
| - """
|
| - self.message_class = message_class
|
| -
|
| - def get_attributes(self, attributes='All'):
|
| - """
|
| - Retrieves attributes about this queue object and returns
|
| - them in an Attribute instance (subclass of a Dictionary).
|
| -
|
| - :type attributes: string
|
| - :param attributes: String containing one of:
|
| - ApproximateNumberOfMessages,
|
| - ApproximateNumberOfMessagesNotVisible,
|
| - VisibilityTimeout,
|
| - CreatedTimestamp,
|
| - LastModifiedTimestamp,
|
| - Policy
|
| - ReceiveMessageWaitTimeSeconds
|
| - :rtype: Attribute object
|
| - :return: An Attribute object which is a mapping type holding the
|
| - requested name/value pairs
|
| - """
|
| - return self.connection.get_queue_attributes(self, attributes)
|
| -
|
| - def set_attribute(self, attribute, value):
|
| - """
|
| - Set a new value for an attribute of the Queue.
|
| -
|
| - :type attribute: String
|
| - :param attribute: The name of the attribute you want to set. The
|
| - only valid value at this time is: VisibilityTimeout
|
| - :type value: int
|
| - :param value: The new value for the attribute.
|
| - For VisibilityTimeout the value must be an
|
| - integer number of seconds from 0 to 86400.
|
| -
|
| - :rtype: bool
|
| - :return: True if successful, otherwise False.
|
| - """
|
| - return self.connection.set_queue_attribute(self, attribute, value)
|
| -
|
| - def get_timeout(self):
|
| - """
|
| - Get the visibility timeout for the queue.
|
| -
|
| - :rtype: int
|
| - :return: The number of seconds as an integer.
|
| - """
|
| - a = self.get_attributes('VisibilityTimeout')
|
| - return int(a['VisibilityTimeout'])
|
| -
|
| - def set_timeout(self, visibility_timeout):
|
| - """
|
| - Set the visibility timeout for the queue.
|
| -
|
| - :type visibility_timeout: int
|
| - :param visibility_timeout: The desired timeout in seconds
|
| - """
|
| - retval = self.set_attribute('VisibilityTimeout', visibility_timeout)
|
| - if retval:
|
| - self.visibility_timeout = visibility_timeout
|
| - return retval
|
| -
|
| - def add_permission(self, label, aws_account_id, action_name):
|
| - """
|
| - Add a permission to a queue.
|
| -
|
| - :type label: str or unicode
|
| - :param label: A unique identification of the permission you are setting.
|
| - Maximum of 80 characters ``[0-9a-zA-Z_-]``
|
| - Example, AliceSendMessage
|
| -
|
| - :type aws_account_id: str or unicode
|
| - :param principal_id: The AWS account number of the principal who
|
| - will be given permission. The principal must have an AWS account,
|
| - but does not need to be signed up for Amazon SQS. For information
|
| - about locating the AWS account identification.
|
| -
|
| - :type action_name: str or unicode
|
| - :param action_name: The action. Valid choices are:
|
| - SendMessage|ReceiveMessage|DeleteMessage|
|
| - ChangeMessageVisibility|GetQueueAttributes|*
|
| -
|
| - :rtype: bool
|
| - :return: True if successful, False otherwise.
|
| -
|
| - """
|
| - return self.connection.add_permission(self, label, aws_account_id,
|
| - action_name)
|
| -
|
| - def remove_permission(self, label):
|
| - """
|
| - Remove a permission from a queue.
|
| -
|
| - :type label: str or unicode
|
| - :param label: The unique label associated with the permission
|
| - being removed.
|
| -
|
| - :rtype: bool
|
| - :return: True if successful, False otherwise.
|
| - """
|
| - return self.connection.remove_permission(self, label)
|
| -
|
| - def read(self, visibility_timeout=None, wait_time_seconds=None,
|
| - message_attributes=None):
|
| - """
|
| - Read a single message from the queue.
|
| -
|
| - :type visibility_timeout: int
|
| - :param visibility_timeout: The timeout for this message in seconds
|
| -
|
| - :type wait_time_seconds: int
|
| - :param wait_time_seconds: The duration (in seconds) for which the call
|
| - will wait for a message to arrive in the queue before returning.
|
| - If a message is available, the call will return sooner than
|
| - wait_time_seconds.
|
| -
|
| - :type message_attributes: list
|
| - :param message_attributes: The name(s) of additional message
|
| - attributes to return. The default is to return no additional
|
| - message attributes. Use ``['All']`` or ``['.*']`` to return all.
|
| -
|
| - :rtype: :class:`boto.sqs.message.Message`
|
| - :return: A single message or None if queue is empty
|
| - """
|
| - rs = self.get_messages(1, visibility_timeout,
|
| - wait_time_seconds=wait_time_seconds,
|
| - message_attributes=message_attributes)
|
| - if len(rs) == 1:
|
| - return rs[0]
|
| - else:
|
| - return None
|
| -
|
| - def write(self, message, delay_seconds=None):
|
| - """
|
| - Add a single message to the queue.
|
| -
|
| - :type message: Message
|
| - :param message: The message to be written to the queue
|
| -
|
| - :rtype: :class:`boto.sqs.message.Message`
|
| - :return: The :class:`boto.sqs.message.Message` object that was written.
|
| - """
|
| - new_msg = self.connection.send_message(self,
|
| - message.get_body_encoded(), delay_seconds=delay_seconds,
|
| - message_attributes=message.message_attributes)
|
| - message.id = new_msg.id
|
| - message.md5 = new_msg.md5
|
| - return message
|
| -
|
| - def write_batch(self, messages):
|
| - """
|
| - Delivers up to 10 messages in a single request.
|
| -
|
| - :type messages: List of lists.
|
| - :param messages: A list of lists or tuples. Each inner
|
| - tuple represents a single message to be written
|
| - and consists of and ID (string) that must be unique
|
| - within the list of messages, the message body itself
|
| - which can be a maximum of 64K in length, an
|
| - integer which represents the delay time (in seconds)
|
| - for the message (0-900) before the message will
|
| - be delivered to the queue, and an optional dict of
|
| - message attributes like those passed to ``send_message``
|
| - in the connection class.
|
| - """
|
| - return self.connection.send_message_batch(self, messages)
|
| -
|
| - def new_message(self, body='', **kwargs):
|
| - """
|
| - Create new message of appropriate class.
|
| -
|
| - :type body: message body
|
| - :param body: The body of the newly created message (optional).
|
| -
|
| - :rtype: :class:`boto.sqs.message.Message`
|
| - :return: A new Message object
|
| - """
|
| - m = self.message_class(self, body, **kwargs)
|
| - m.queue = self
|
| - return m
|
| -
|
| - # get a variable number of messages, returns a list of messages
|
| - def get_messages(self, num_messages=1, visibility_timeout=None,
|
| - attributes=None, wait_time_seconds=None,
|
| - message_attributes=None):
|
| - """
|
| - Get a variable number of messages.
|
| -
|
| - :type num_messages: int
|
| - :param num_messages: The maximum number of messages to read from
|
| - the queue.
|
| -
|
| - :type visibility_timeout: int
|
| - :param visibility_timeout: The VisibilityTimeout for the messages read.
|
| -
|
| - :type attributes: str
|
| - :param attributes: The name of additional attribute to return
|
| - with response or All if you want all attributes. The
|
| - default is to return no additional attributes. Valid
|
| - values: All SenderId SentTimestamp ApproximateReceiveCount
|
| - ApproximateFirstReceiveTimestamp
|
| -
|
| - :type wait_time_seconds: int
|
| - :param wait_time_seconds: The duration (in seconds) for which the call
|
| - will wait for a message to arrive in the queue before returning.
|
| - If a message is available, the call will return sooner than
|
| - wait_time_seconds.
|
| -
|
| - :type message_attributes: list
|
| - :param message_attributes: The name(s) of additional message
|
| - attributes to return. The default is to return no additional
|
| - message attributes. Use ``['All']`` or ``['.*']`` to return all.
|
| -
|
| - :rtype: list
|
| - :return: A list of :class:`boto.sqs.message.Message` objects.
|
| - """
|
| - return self.connection.receive_message(
|
| - self, number_messages=num_messages,
|
| - visibility_timeout=visibility_timeout, attributes=attributes,
|
| - wait_time_seconds=wait_time_seconds,
|
| - message_attributes=message_attributes)
|
| -
|
| - def delete_message(self, message):
|
| - """
|
| - Delete a message from the queue.
|
| -
|
| - :type message: :class:`boto.sqs.message.Message`
|
| - :param message: The :class:`boto.sqs.message.Message` object to delete.
|
| -
|
| - :rtype: bool
|
| - :return: True if successful, False otherwise
|
| - """
|
| - return self.connection.delete_message(self, message)
|
| -
|
| - def delete_message_batch(self, messages):
|
| - """
|
| - Deletes a list of messages in a single request.
|
| -
|
| - :type messages: List of :class:`boto.sqs.message.Message` objects.
|
| - :param messages: A list of message objects.
|
| - """
|
| - return self.connection.delete_message_batch(self, messages)
|
| -
|
| - def change_message_visibility_batch(self, messages):
|
| - """
|
| - A batch version of change_message_visibility that can act
|
| - on up to 10 messages at a time.
|
| -
|
| - :type messages: List of tuples.
|
| - :param messages: A list of tuples where each tuple consists
|
| - of a :class:`boto.sqs.message.Message` object and an integer
|
| - that represents the new visibility timeout for that message.
|
| - """
|
| - return self.connection.change_message_visibility_batch(self, messages)
|
| -
|
| - def delete(self):
|
| - """
|
| - Delete the queue.
|
| - """
|
| - return self.connection.delete_queue(self)
|
| -
|
| - def purge(self):
|
| - """
|
| - Purge all messages in the queue.
|
| - """
|
| - return self.connection.purge_queue(self)
|
| -
|
| - def clear(self, page_size=10, vtimeout=10):
|
| - """Deprecated utility function to remove all messages from a queue"""
|
| - return self.purge()
|
| -
|
| - def count(self, page_size=10, vtimeout=10):
|
| - """
|
| - Utility function to count the number of messages in a queue.
|
| - Note: This function now calls GetQueueAttributes to obtain
|
| - an 'approximate' count of the number of messages in a queue.
|
| - """
|
| - a = self.get_attributes('ApproximateNumberOfMessages')
|
| - return int(a['ApproximateNumberOfMessages'])
|
| -
|
| - def count_slow(self, page_size=10, vtimeout=10):
|
| - """
|
| - Deprecated. This is the old 'count' method that actually counts
|
| - the messages by reading them all. This gives an accurate count but
|
| - is very slow for queues with non-trivial number of messasges.
|
| - Instead, use get_attributes('ApproximateNumberOfMessages') to take
|
| - advantage of the new SQS capability. This is retained only for
|
| - the unit tests.
|
| - """
|
| - n = 0
|
| - l = self.get_messages(page_size, vtimeout)
|
| - while l:
|
| - for m in l:
|
| - n += 1
|
| - l = self.get_messages(page_size, vtimeout)
|
| - return n
|
| -
|
| - def dump(self, file_name, page_size=10, vtimeout=10, sep='\n'):
|
| - """Utility function to dump the messages in a queue to a file
|
| - NOTE: Page size must be < 10 else SQS errors"""
|
| - fp = open(file_name, 'wb')
|
| - n = 0
|
| - l = self.get_messages(page_size, vtimeout)
|
| - while l:
|
| - for m in l:
|
| - fp.write(m.get_body())
|
| - if sep:
|
| - fp.write(sep)
|
| - n += 1
|
| - l = self.get_messages(page_size, vtimeout)
|
| - fp.close()
|
| - return n
|
| -
|
| - def save_to_file(self, fp, sep='\n'):
|
| - """
|
| - Read all messages from the queue and persist them to file-like object.
|
| - Messages are written to the file and the 'sep' string is written
|
| - in between messages. Messages are deleted from the queue after
|
| - being written to the file.
|
| - Returns the number of messages saved.
|
| - """
|
| - n = 0
|
| - m = self.read()
|
| - while m:
|
| - n += 1
|
| - fp.write(m.get_body())
|
| - if sep:
|
| - fp.write(sep)
|
| - self.delete_message(m)
|
| - m = self.read()
|
| - return n
|
| -
|
| - def save_to_filename(self, file_name, sep='\n'):
|
| - """
|
| - Read all messages from the queue and persist them to local file.
|
| - Messages are written to the file and the 'sep' string is written
|
| - in between messages. Messages are deleted from the queue after
|
| - being written to the file.
|
| - Returns the number of messages saved.
|
| - """
|
| - fp = open(file_name, 'wb')
|
| - n = self.save_to_file(fp, sep)
|
| - fp.close()
|
| - return n
|
| -
|
| - # for backwards compatibility
|
| - save = save_to_filename
|
| -
|
| - def save_to_s3(self, bucket):
|
| - """
|
| - Read all messages from the queue and persist them to S3.
|
| - Messages are stored in the S3 bucket using a naming scheme of::
|
| -
|
| - <queue_id>/<message_id>
|
| -
|
| - Messages are deleted from the queue after being saved to S3.
|
| - Returns the number of messages saved.
|
| - """
|
| - n = 0
|
| - m = self.read()
|
| - while m:
|
| - n += 1
|
| - key = bucket.new_key('%s/%s' % (self.id, m.id))
|
| - key.set_contents_from_string(m.get_body())
|
| - self.delete_message(m)
|
| - m = self.read()
|
| - return n
|
| -
|
| - def load_from_s3(self, bucket, prefix=None):
|
| - """
|
| - Load messages previously saved to S3.
|
| - """
|
| - n = 0
|
| - if prefix:
|
| - prefix = '%s/' % prefix
|
| - else:
|
| - prefix = '%s/' % self.id[1:]
|
| - rs = bucket.list(prefix=prefix)
|
| - for key in rs:
|
| - n += 1
|
| - m = self.new_message(key.get_contents_as_string())
|
| - self.write(m)
|
| - return n
|
| -
|
| - def load_from_file(self, fp, sep='\n'):
|
| - """Utility function to load messages from a file-like object to a queue"""
|
| - n = 0
|
| - body = ''
|
| - l = fp.readline()
|
| - while l:
|
| - if l == sep:
|
| - m = Message(self, body)
|
| - self.write(m)
|
| - n += 1
|
| - print('writing message %d' % n)
|
| - body = ''
|
| - else:
|
| - body = body + l
|
| - l = fp.readline()
|
| - return n
|
| -
|
| - def load_from_filename(self, file_name, sep='\n'):
|
| - """Utility function to load messages from a local filename to a queue"""
|
| - fp = open(file_name, 'rb')
|
| - n = self.load_from_file(fp, sep)
|
| - fp.close()
|
| - return n
|
| -
|
| - # for backward compatibility
|
| - load = load_from_filename
|
| -
|
|
|