| OLD | NEW |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Classes representing individual metrics that can be sent.""" | 5 """Classes representing individual metrics that can be sent.""" |
| 6 | 6 |
| 7 import copy | 7 import copy |
| 8 import re | 8 import re |
| 9 | 9 |
| 10 from infra_libs.ts_mon.protos.current import metrics_pb2 | 10 from infra_libs.ts_mon.protos import metrics_pb2 |
| 11 from infra_libs.ts_mon.protos.new import metrics_pb2 as new_metrics_pb2 | |
| 12 | 11 |
| 13 from infra_libs.ts_mon.common import distribution | 12 from infra_libs.ts_mon.common import distribution |
| 14 from infra_libs.ts_mon.common import errors | 13 from infra_libs.ts_mon.common import errors |
| 15 from infra_libs.ts_mon.common import interface | 14 from infra_libs.ts_mon.common import interface |
| 16 | 15 |
| 17 | 16 |
| 18 MICROSECONDS_PER_SECOND = 1000000 | 17 MICROSECONDS_PER_SECOND = 1000000 |
| 19 | 18 |
| 20 | 19 |
| 21 class Field(object): | 20 class Field(object): |
| 22 FIELD_NAME_PATTERN = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') | 21 FIELD_NAME_PATTERN = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') |
| 23 | 22 |
| 24 allowed_python_types = None | 23 allowed_python_types = None |
| 25 v1_type = None | 24 type_enum = None |
| 26 v2_type = None | 25 field_name = None |
| 27 v1_field = None | |
| 28 v2_field = None | |
| 29 | 26 |
| 30 def __init__(self, name): | 27 def __init__(self, name): |
| 31 if not self.FIELD_NAME_PATTERN.match(name): | 28 if not self.FIELD_NAME_PATTERN.match(name): |
| 32 raise errors.MetricDefinitionError( | 29 raise errors.MetricDefinitionError( |
| 33 'Invalid metric field name "%s" - must match the regex "%s"' % ( | 30 'Invalid metric field name "%s" - must match the regex "%s"' % ( |
| 34 name, self.FIELD_NAME_PATTERN.pattern)) | 31 name, self.FIELD_NAME_PATTERN.pattern)) |
| 35 | 32 |
| 36 self.name = name | 33 self.name = name |
| 37 | 34 |
| 38 def validate_value(self, metric_name, value): | 35 def validate_value(self, metric_name, value): |
| 39 if not isinstance(value, self.allowed_python_types): | 36 if not isinstance(value, self.allowed_python_types): |
| 40 raise errors.MonitoringInvalidFieldTypeError( | 37 raise errors.MonitoringInvalidFieldTypeError( |
| 41 metric_name, self.name, value) | 38 metric_name, self.name, value) |
| 42 | 39 |
| 43 def populate_proto_v1(self, proto, value): | 40 def populate_proto(self, proto, value): |
| 44 setattr(proto, self.v1_field, value) | 41 setattr(proto, self.field_name, value) |
| 45 | |
| 46 def populate_proto_v2(self, proto, value): | |
| 47 setattr(proto, self.v2_field, value) | |
| 48 | 42 |
| 49 | 43 |
| 50 class StringField(Field): | 44 class StringField(Field): |
| 51 allowed_python_types = basestring | 45 allowed_python_types = basestring |
| 52 v1_type = metrics_pb2.MetricsField.STRING | 46 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.STRING |
| 53 v2_type = new_metrics_pb2.MetricsDataSet.MetricFieldDescriptor.STRING | 47 field_name = 'string_value' |
| 54 v1_field = 'string_value' | |
| 55 v2_field = 'string_value' | |
| 56 | 48 |
| 57 | 49 |
| 58 class IntegerField(Field): | 50 class IntegerField(Field): |
| 59 allowed_python_types = (int, long) | 51 allowed_python_types = (int, long) |
| 60 v1_type = metrics_pb2.MetricsField.INT | 52 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.INT64 |
| 61 v2_type = new_metrics_pb2.MetricsDataSet.MetricFieldDescriptor.INT64 | 53 field_name = 'int64_value' |
| 62 v1_field = 'int_value' | |
| 63 v2_field = 'int64_value' | |
| 64 | 54 |
| 65 | 55 |
| 66 class BooleanField(Field): | 56 class BooleanField(Field): |
| 67 allowed_python_types = bool | 57 allowed_python_types = bool |
| 68 v1_type = metrics_pb2.MetricsField.BOOL | 58 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.BOOL |
| 69 v2_type = new_metrics_pb2.MetricsDataSet.MetricFieldDescriptor.BOOL | 59 field_name = 'bool_value' |
| 70 v1_field = 'bool_value' | |
| 71 v2_field = 'bool_value' | |
| 72 | 60 |
| 73 | 61 |
| 74 class Metric(object): | 62 class Metric(object): |
| 75 """Abstract base class for a metric. | 63 """Abstract base class for a metric. |
| 76 | 64 |
| 77 A Metric is an attribute that may be monitored across many targets. Examples | 65 A Metric is an attribute that may be monitored across many targets. Examples |
| 78 include disk usage or the number of requests a server has received. A single | 66 include disk usage or the number of requests a server has received. A single |
| 79 process may keep track of many metrics. | 67 process may keep track of many metrics. |
| 80 | 68 |
| 81 Note that Metric objects may be initialized at any time (for example, at the | 69 Note that Metric objects may be initialized at any time (for example, at the |
| 82 top of a library), but cannot be sent until the underlying Monitor object | 70 top of a library), but cannot be sent until the underlying Monitor object |
| 83 has been set up (usually by the top-level process parsing the command line). | 71 has been set up (usually by the top-level process parsing the command line). |
| 84 | 72 |
| 85 A Metric can actually store multiple values that are identified by a set of | 73 A Metric can actually store multiple values that are identified by a set of |
| 86 fields (which are themselves key-value pairs). Fields can be passed to the | 74 fields (which are themselves key-value pairs). Fields can be passed to the |
| 87 set() or increment() methods to modify a particular value, or passed to the | 75 set() or increment() methods to modify a particular value, or passed to the |
| 88 constructor in which case they will be used as the defaults for this Metric. | 76 constructor in which case they will be used as the defaults for this Metric. |
| 89 | 77 |
| 90 The unit of measurement for Metric data can be specified with MetricsDataUnits | 78 The unit of measurement for Metric data should be specified with |
| 91 when a Metric object is created: | 79 MetricsDataUnits when a Metric object is created: |
| 92 e.g., MetricsDataUnits.SECONDS, MetricsDataUnits.BYTES, and etc.., | 80 e.g., MetricsDataUnits.SECONDS, MetricsDataUnits.BYTES, and etc.., |
| 93 A full list of supported units can be found in the following protobuf file | 81 See `MetricsDataUnits` class for a full list of units. |
| 94 : infra_libs/ts_mon/protos/metrics.proto | |
| 95 | 82 |
| 96 Do not directly instantiate an object of this class. | 83 Do not directly instantiate an object of this class. |
| 97 Use the concrete child classes instead: | 84 Use the concrete child classes instead: |
| 98 * StringMetric for metrics with string value | 85 * StringMetric for metrics with string value |
| 99 * BooleanMetric for metrics with boolean values | 86 * BooleanMetric for metrics with boolean values |
| 100 * CounterMetric for metrics with monotonically increasing integer values | 87 * CounterMetric for metrics with monotonically increasing integer values |
| 101 * GaugeMetric for metrics with arbitrarily varying integer values | 88 * GaugeMetric for metrics with arbitrarily varying integer values |
| 102 * CumulativeMetric for metrics with monotonically increasing float values | 89 * CumulativeMetric for metrics with monotonically increasing float values |
| 103 * FloatMetric for metrics with arbitrarily varying float values | 90 * FloatMetric for metrics with arbitrarily varying float values |
| 104 | 91 |
| 105 See http://go/inframon-doc for help designing and using your metrics. | 92 See http://go/inframon-doc for help designing and using your metrics. |
| 106 """ | 93 """ |
| 107 | 94 |
| 108 def __init__(self, name, description, field_spec, units=None): | 95 def __init__(self, name, description, field_spec, units=None): |
| 109 """Create an instance of a Metric. | 96 """Create an instance of a Metric. |
| 110 | 97 |
| 111 Args: | 98 Args: |
| 112 name (str): the file-like name of this metric | 99 name (str): the file-like name of this metric |
| 113 description (string): help string for the metric. Should be enough to | 100 description (string): help string for the metric. Should be enough to |
| 114 know what the metric is about. | 101 know what the metric is about. |
| 115 field_spec (list): a list of Field subclasses to define the fields that | 102 field_spec (list): a list of Field subclasses to define the fields that |
| 116 are allowed on this metric. Pass a list of either | 103 are allowed on this metric. Pass a list of either |
| 117 StringField, IntegerField or BooleanField here. | 104 StringField, IntegerField or BooleanField here. |
| 118 units (int): the unit used to measure data for given | 105 units (string): the unit used to measure data for given metric. Some |
| 119 metric. Please use the attributes of MetricDataUnit to find | 106 common units are pre-defined in the MetricsDataUnits |
| 120 valid integer values for this argument. | 107 class. |
| 121 """ | 108 """ |
| 122 field_spec = field_spec or [] | 109 field_spec = field_spec or [] |
| 123 | 110 |
| 124 self._name = name.lstrip('/') | 111 self._name = name.lstrip('/') |
| 125 | 112 |
| 126 if not isinstance(description, basestring): | 113 if not isinstance(description, basestring): |
| 127 raise errors.MetricDefinitionError('Metric description must be a string') | 114 raise errors.MetricDefinitionError('Metric description must be a string') |
| 128 if not description: | 115 if not description: |
| 129 raise errors.MetricDefinitionError('Metric must have a description') | 116 raise errors.MetricDefinitionError('Metric must have a description') |
| 130 if (not isinstance(field_spec, (list, tuple)) or | 117 if (not isinstance(field_spec, (list, tuple)) or |
| 131 any(not isinstance(x, Field) for x in field_spec)): | 118 any(not isinstance(x, Field) for x in field_spec)): |
| 132 raise errors.MetricDefinitionError( | 119 raise errors.MetricDefinitionError( |
| 133 'Metric constructor takes a list of Fields, or None') | 120 'Metric constructor takes a list of Fields, or None') |
| 134 if len(field_spec) > 7: | 121 if len(field_spec) > 7: |
| 135 raise errors.MonitoringTooManyFieldsError(self._name, field_spec) | 122 raise errors.MonitoringTooManyFieldsError(self._name, field_spec) |
| 136 | 123 |
| 137 self._start_time = None | 124 self._start_time = None |
| 138 self._field_spec = field_spec | 125 self._field_spec = field_spec |
| 139 self._sorted_field_names = sorted(x.name for x in field_spec) | 126 self._sorted_field_names = sorted(x.name for x in field_spec) |
| 140 self._description = description | 127 self._description = description |
| 141 self._units = units | 128 self._units = units |
| 142 | 129 |
| 143 interface.register(self) | 130 interface.register(self) |
| 144 | 131 |
| 145 @property | 132 @property |
| 133 def field_spec(self): |
| 134 return list(self._field_spec) |
| 135 |
| 136 @property |
| 146 def name(self): | 137 def name(self): |
| 147 return self._name | 138 return self._name |
| 148 | 139 |
| 149 @property | 140 @property |
| 150 def start_time(self): | 141 def start_time(self): |
| 151 return self._start_time | 142 return self._start_time |
| 152 | 143 |
| 144 @property |
| 145 def units(self): |
| 146 return self._units |
| 147 |
| 153 def is_cumulative(self): | 148 def is_cumulative(self): |
| 154 raise NotImplementedError() | 149 raise NotImplementedError() |
| 155 | 150 |
| 156 def unregister(self): | 151 def unregister(self): |
| 157 interface.unregister(self) | 152 interface.unregister(self) |
| 158 | 153 |
| 159 @staticmethod | 154 def populate_data_set(self, data_set): |
| 160 def _map_units_to_string(units): | |
| 161 """Map MetricsDataUnits to the corresponding string according to: | |
| 162 http://unitsofmeasure.org/ucum.html because that's what the new proto | |
| 163 requires.""" | |
| 164 if units in _UNITS_TO_STRING: | |
| 165 return _UNITS_TO_STRING[units] | |
| 166 else: | |
| 167 return '{unknown}' | |
| 168 | |
| 169 def _populate_data_set(self, data_set): | |
| 170 """Populate MetricsDataSet.""" | 155 """Populate MetricsDataSet.""" |
| 171 data_set.metric_name = '%s%s' % (interface.state.metric_name_prefix, | 156 data_set.metric_name = '%s%s' % (interface.state.metric_name_prefix, |
| 172 self._name) | 157 self._name) |
| 173 data_set.description = self._description or '' | 158 data_set.description = self._description or '' |
| 174 data_set.annotations.unit = self._map_units_to_string(self._units) | 159 if self._units is not None: |
| 160 data_set.annotations.unit = self._units |
| 175 | 161 |
| 176 if self.is_cumulative(): | 162 if self.is_cumulative(): |
| 177 data_set.stream_kind = new_metrics_pb2.CUMULATIVE | 163 data_set.stream_kind = metrics_pb2.CUMULATIVE |
| 178 else: | 164 else: |
| 179 data_set.stream_kind = new_metrics_pb2.GAUGE | 165 data_set.stream_kind = metrics_pb2.GAUGE |
| 180 | 166 |
| 181 self._populate_value_type(data_set) | 167 self._populate_value_type(data_set) |
| 182 self._populate_field_descriptors(data_set) | 168 self._populate_field_descriptors(data_set) |
| 183 | 169 |
| 184 def _populate_data(self, data, start_time, end_time, fields, value): | 170 def populate_data(self, data, start_time, end_time, fields, value): |
| 185 """Populate a new metrics_pb2.MetricsData. | 171 """Populate a new metrics_pb2.MetricsData. |
| 186 | 172 |
| 187 Args: | 173 Args: |
| 188 data_ (new_metrics_pb2.MetricsData): protocol buffer into | 174 data (metrics_pb2.MetricsData): protocol buffer into |
| 189 which to populate the current metric values. | 175 which to populate the current metric values. |
| 190 start_time (int): timestamp in microseconds since UNIX epoch. | 176 start_time (int): timestamp in microseconds since UNIX epoch. |
| 191 """ | 177 """ |
| 192 data.start_timestamp.seconds = int(start_time) | 178 data.start_timestamp.seconds = int(start_time) |
| 193 data.end_timestamp.seconds = int(end_time) | 179 data.end_timestamp.seconds = int(end_time) |
| 194 | 180 |
| 195 self._populate_fields_new(data, fields) | 181 self._populate_fields(data, fields) |
| 196 self._populate_value_new(data, value) | 182 self._populate_value(data, value) |
| 197 | |
| 198 def serialize_to(self, metric_pb, start_time, fields, value, target): | |
| 199 """Generate metrics_pb2.MetricsData messages for this metric. | |
| 200 | |
| 201 Args: | |
| 202 metric_pb (metrics_pb2.MetricsData): protocol buffer into which | |
| 203 to serialize the current metric values. | |
| 204 start_time (int): timestamp in microseconds since UNIX epoch. | |
| 205 target (Target): a Target to use. | |
| 206 """ | |
| 207 | |
| 208 metric_pb.metric_name_prefix = interface.state.metric_name_prefix | |
| 209 metric_pb.name = self._name | |
| 210 metric_pb.description = self._description | |
| 211 if self._units is not None: | |
| 212 metric_pb.units = self._units | |
| 213 | |
| 214 self._populate_value(metric_pb, value, start_time) | |
| 215 self._populate_fields(metric_pb, fields) | |
| 216 | |
| 217 target._populate_target_pb(metric_pb) | |
| 218 | 183 |
| 219 def _populate_field_descriptors(self, data_set): | 184 def _populate_field_descriptors(self, data_set): |
| 220 """Populate `field_descriptor` in MetricsDataSet. | 185 """Populate `field_descriptor` in MetricsDataSet. |
| 221 | 186 |
| 222 Args: | 187 Args: |
| 223 data_set (new_metrics_pb2.MetricsDataSet): a data set protobuf to populate | 188 data_set (metrics_pb2.MetricsDataSet): a data set protobuf to populate |
| 224 """ | 189 """ |
| 225 for spec in self._field_spec: | 190 for spec in self._field_spec: |
| 226 descriptor = data_set.field_descriptor.add() | 191 descriptor = data_set.field_descriptor.add() |
| 227 descriptor.name = spec.name | 192 descriptor.name = spec.name |
| 228 descriptor.field_type = spec.v2_type | 193 descriptor.field_type = spec.type_enum |
| 229 | 194 |
| 230 def _populate_fields_new(self, data, field_values): | 195 def _populate_fields(self, data, field_values): |
| 231 """Fill in the fields attribute of a metric protocol buffer. | 196 """Fill in the fields attribute of a metric protocol buffer. |
| 232 | 197 |
| 233 Args: | 198 Args: |
| 234 metric (metrics_pb2.MetricsData): a metrics protobuf to populate | 199 metric (metrics_pb2.MetricsData): a metrics protobuf to populate |
| 235 field_values (tuple): field values | 200 field_values (tuple): field values |
| 236 """ | 201 """ |
| 237 for spec, value in zip(self._field_spec, field_values): | 202 for spec, value in zip(self._field_spec, field_values): |
| 238 field = data.field.add() | 203 field = data.field.add() |
| 239 field.name = spec.name | 204 field.name = spec.name |
| 240 spec.populate_proto_v2(field, value) | 205 spec.populate_proto(field, value) |
| 241 | |
| 242 def _populate_fields(self, metric, field_values): | |
| 243 """Fill in the fields attribute of a metric protocol buffer. | |
| 244 | |
| 245 Args: | |
| 246 metric (metrics_pb2.MetricsData): a metrics protobuf to populate | |
| 247 field_values (tuple): field values | |
| 248 """ | |
| 249 for spec, value in zip(self._field_spec, field_values): | |
| 250 field = metric.fields.add() | |
| 251 field.name = spec.name | |
| 252 field.type = spec.v1_type | |
| 253 spec.populate_proto_v1(field, value) | |
| 254 | 206 |
| 255 def _validate_fields(self, fields): | 207 def _validate_fields(self, fields): |
| 256 """Checks the correct number and types of field values were provided. | 208 """Checks the correct number and types of field values were provided. |
| 257 | 209 |
| 258 Args: | 210 Args: |
| 259 fields (dict): A dict of field values given by the user, or None. | 211 fields (dict): A dict of field values given by the user, or None. |
| 260 | 212 |
| 261 Returns: | 213 Returns: |
| 262 fields' values as a tuple, in the same order as the field_spec. | 214 fields' values as a tuple, in the same order as the field_spec. |
| 263 | 215 |
| (...skipping 11 matching lines...) Expand all Loading... |
| 275 | 227 |
| 276 if sorted(fields) != self._sorted_field_names: | 228 if sorted(fields) != self._sorted_field_names: |
| 277 raise errors.WrongFieldsError( | 229 raise errors.WrongFieldsError( |
| 278 self.name, fields.keys(), self._sorted_field_names) | 230 self.name, fields.keys(), self._sorted_field_names) |
| 279 | 231 |
| 280 for spec in self._field_spec: | 232 for spec in self._field_spec: |
| 281 spec.validate_value(self.name, fields[spec.name]) | 233 spec.validate_value(self.name, fields[spec.name]) |
| 282 | 234 |
| 283 return tuple(fields[spec.name] for spec in self._field_spec) | 235 return tuple(fields[spec.name] for spec in self._field_spec) |
| 284 | 236 |
| 285 def _populate_value(self, metric, value, start_time): | 237 def _populate_value(self, data, value): |
| 286 """Fill in the the data values of a metric protocol buffer. | 238 """Fill in the the data values of a metric protocol buffer. |
| 287 | 239 |
| 288 Args: | 240 Args: |
| 289 metric (metrics_pb2.MetricsData): a metrics protobuf to populate | |
| 290 value (see concrete class): the value of the metric to be set | |
| 291 start_time (int): timestamp in microseconds since UNIX epoch. | |
| 292 """ | |
| 293 raise NotImplementedError() | |
| 294 | |
| 295 def _populate_value_new(self, data, value): | |
| 296 """Fill in the the data values of a metric protocol buffer. | |
| 297 | |
| 298 Args: | |
| 299 data (metrics_pb2.MetricsData): a metrics protobuf to populate | 241 data (metrics_pb2.MetricsData): a metrics protobuf to populate |
| 300 value (see concrete class): the value of the metric to be set | 242 value (see concrete class): the value of the metric to be set |
| 301 """ | 243 """ |
| 302 raise NotImplementedError() | 244 raise NotImplementedError() |
| 303 | 245 |
| 304 def _populate_value_type(self, data_set): | 246 def _populate_value_type(self, data_set): |
| 305 """Fill in the the data values of a metric protocol buffer. | 247 """Fill in the the data values of a metric protocol buffer. |
| 306 | 248 |
| 307 Args: | 249 Args: |
| 308 data_set (metrics_pb2.MetricsDataSet): a MetricsDataSet protobuf to | 250 data_set (metrics_pb2.MetricsDataSet): a MetricsDataSet protobuf to |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 351 | 293 |
| 352 def _incr(self, fields, target_fields, delta, modify_fn=None): | 294 def _incr(self, fields, target_fields, delta, modify_fn=None): |
| 353 interface.state.store.incr( | 295 interface.state.store.incr( |
| 354 self.name, self._validate_fields(fields), target_fields, | 296 self.name, self._validate_fields(fields), target_fields, |
| 355 delta, modify_fn=modify_fn) | 297 delta, modify_fn=modify_fn) |
| 356 | 298 |
| 357 | 299 |
| 358 class StringMetric(Metric): | 300 class StringMetric(Metric): |
| 359 """A metric whose value type is a string.""" | 301 """A metric whose value type is a string.""" |
| 360 | 302 |
| 361 def _populate_value(self, metric, value, start_time): | 303 def _populate_value(self, data, value): |
| 362 metric.string_value = value | |
| 363 | |
| 364 def _populate_value_new(self, data, value): | |
| 365 data.string_value = value | 304 data.string_value = value |
| 366 | 305 |
| 367 def _populate_value_type(self, data_set): | 306 def _populate_value_type(self, data_set): |
| 368 data_set.value_type = new_metrics_pb2.STRING | 307 data_set.value_type = metrics_pb2.STRING |
| 369 | 308 |
| 370 def set(self, value, fields=None, target_fields=None): | 309 def set(self, value, fields=None, target_fields=None): |
| 371 if not isinstance(value, basestring): | 310 if not isinstance(value, basestring): |
| 372 raise errors.MonitoringInvalidValueTypeError(self._name, value) | 311 raise errors.MonitoringInvalidValueTypeError(self._name, value) |
| 373 self._set(fields, target_fields, value) | 312 self._set(fields, target_fields, value) |
| 374 | 313 |
| 375 def is_cumulative(self): | 314 def is_cumulative(self): |
| 376 return False | 315 return False |
| 377 | 316 |
| 378 | 317 |
| 379 class BooleanMetric(Metric): | 318 class BooleanMetric(Metric): |
| 380 """A metric whose value type is a boolean.""" | 319 """A metric whose value type is a boolean.""" |
| 381 | 320 |
| 382 def _populate_value(self, metric, value, start_time): | 321 def _populate_value(self, data, value): |
| 383 metric.boolean_value = value | |
| 384 | |
| 385 def _populate_value_new(self, data, value): | |
| 386 data.bool_value = value | 322 data.bool_value = value |
| 387 | 323 |
| 388 def _populate_value_type(self, data_set): | 324 def _populate_value_type(self, data_set): |
| 389 data_set.value_type = new_metrics_pb2.BOOL | 325 data_set.value_type = metrics_pb2.BOOL |
| 390 | 326 |
| 391 def set(self, value, fields=None, target_fields=None): | 327 def set(self, value, fields=None, target_fields=None): |
| 392 if not isinstance(value, bool): | 328 if not isinstance(value, bool): |
| 393 raise errors.MonitoringInvalidValueTypeError(self._name, value) | 329 raise errors.MonitoringInvalidValueTypeError(self._name, value) |
| 394 self._set(fields, target_fields, value) | 330 self._set(fields, target_fields, value) |
| 395 | 331 |
| 396 def is_cumulative(self): | 332 def is_cumulative(self): |
| 397 return False | 333 return False |
| 398 | 334 |
| 399 | 335 |
| 400 class NumericMetric(Metric): # pylint: disable=abstract-method | 336 class NumericMetric(Metric): # pylint: disable=abstract-method |
| 401 """Abstract base class for numeric (int or float) metrics.""" | 337 """Abstract base class for numeric (int or float) metrics.""" |
| 402 | 338 |
| 403 def increment(self, fields=None, target_fields=None): | 339 def increment(self, fields=None, target_fields=None): |
| 404 self._incr(fields, target_fields, 1) | 340 self._incr(fields, target_fields, 1) |
| 405 | 341 |
| 406 def increment_by(self, step, fields=None, target_fields=None): | 342 def increment_by(self, step, fields=None, target_fields=None): |
| 407 self._incr(fields, target_fields, step) | 343 self._incr(fields, target_fields, step) |
| 408 | 344 |
| 409 | 345 |
| 410 class CounterMetric(NumericMetric): | 346 class CounterMetric(NumericMetric): |
| 411 """A metric whose value type is a monotonically increasing integer.""" | 347 """A metric whose value type is a monotonically increasing integer.""" |
| 412 | 348 |
| 413 def __init__(self, name, description, field_spec=None, start_time=None, | 349 def __init__(self, name, description, field_spec, start_time=None, |
| 414 units=None): | 350 units=None): |
| 415 super(CounterMetric, self).__init__( | 351 super(CounterMetric, self).__init__( |
| 416 name, description, field_spec, units=units) | 352 name, description, field_spec, units=units) |
| 417 self._start_time = start_time | 353 self._start_time = start_time |
| 418 | 354 |
| 419 def _populate_value(self, metric, value, start_time): | 355 def _populate_value(self, data, value): |
| 420 metric.counter = value | |
| 421 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND) | |
| 422 | |
| 423 def _populate_value_new(self, data, value): | |
| 424 data.int64_value = value | 356 data.int64_value = value |
| 425 | 357 |
| 426 def _populate_value_type(self, data_set): | 358 def _populate_value_type(self, data_set): |
| 427 data_set.value_type = new_metrics_pb2.INT64 | 359 data_set.value_type = metrics_pb2.INT64 |
| 428 | 360 |
| 429 def set(self, value, fields=None, target_fields=None): | 361 def set(self, value, fields=None, target_fields=None): |
| 430 if not isinstance(value, (int, long)): | 362 if not isinstance(value, (int, long)): |
| 431 raise errors.MonitoringInvalidValueTypeError(self._name, value) | 363 raise errors.MonitoringInvalidValueTypeError(self._name, value) |
| 432 self._set(fields, target_fields, value, enforce_ge=True) | 364 self._set(fields, target_fields, value, enforce_ge=True) |
| 433 | 365 |
| 434 def increment_by(self, step, fields=None, target_fields=None): | 366 def increment_by(self, step, fields=None, target_fields=None): |
| 435 if not isinstance(step, (int, long)): | 367 if not isinstance(step, (int, long)): |
| 436 raise errors.MonitoringInvalidValueTypeError(self._name, step) | 368 raise errors.MonitoringInvalidValueTypeError(self._name, step) |
| 437 self._incr(fields, target_fields, step) | 369 self._incr(fields, target_fields, step) |
| 438 | 370 |
| 439 def is_cumulative(self): | 371 def is_cumulative(self): |
| 440 return True | 372 return True |
| 441 | 373 |
| 442 | 374 |
| 443 class GaugeMetric(NumericMetric): | 375 class GaugeMetric(NumericMetric): |
| 444 """A metric whose value type is an integer.""" | 376 """A metric whose value type is an integer.""" |
| 445 | 377 |
| 446 def _populate_value(self, metric, value, start_time): | 378 def _populate_value(self, data, value): |
| 447 metric.gauge = value | |
| 448 | |
| 449 def _populate_value_new(self, data, value): | |
| 450 data.int64_value = value | 379 data.int64_value = value |
| 451 | 380 |
| 452 def _populate_value_type(self, data_set): | 381 def _populate_value_type(self, data_set): |
| 453 data_set.value_type = new_metrics_pb2.INT64 | 382 data_set.value_type = metrics_pb2.INT64 |
| 454 | 383 |
| 455 def set(self, value, fields=None, target_fields=None): | 384 def set(self, value, fields=None, target_fields=None): |
| 456 if not isinstance(value, (int, long)): | 385 if not isinstance(value, (int, long)): |
| 457 raise errors.MonitoringInvalidValueTypeError(self._name, value) | 386 raise errors.MonitoringInvalidValueTypeError(self._name, value) |
| 458 self._set(fields, target_fields, value) | 387 self._set(fields, target_fields, value) |
| 459 | 388 |
| 460 def is_cumulative(self): | 389 def is_cumulative(self): |
| 461 return False | 390 return False |
| 462 | 391 |
| 463 | 392 |
| 464 class CumulativeMetric(NumericMetric): | 393 class CumulativeMetric(NumericMetric): |
| 465 """A metric whose value type is a monotonically increasing float.""" | 394 """A metric whose value type is a monotonically increasing float.""" |
| 466 | 395 |
| 467 def __init__(self, name, description, field_spec=None, start_time=None, | 396 def __init__(self, name, description, field_spec, start_time=None, |
| 468 units=None): | 397 units=None): |
| 469 super(CumulativeMetric, self).__init__( | 398 super(CumulativeMetric, self).__init__( |
| 470 name, description, field_spec, units=units) | 399 name, description, field_spec, units=units) |
| 471 self._start_time = start_time | 400 self._start_time = start_time |
| 472 | 401 |
| 473 def _populate_value(self, metric, value, start_time): | 402 def _populate_value(self, data, value): |
| 474 metric.cumulative_double_value = value | |
| 475 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND) | |
| 476 | |
| 477 def _populate_value_new(self, data, value): | |
| 478 data.double_value = value | 403 data.double_value = value |
| 479 | 404 |
| 480 def _populate_value_type(self, data_set): | 405 def _populate_value_type(self, data_set): |
| 481 data_set.value_type = new_metrics_pb2.DOUBLE | 406 data_set.value_type = metrics_pb2.DOUBLE |
| 482 | 407 |
| 483 def set(self, value, fields=None, target_fields=None): | 408 def set(self, value, fields=None, target_fields=None): |
| 484 if not isinstance(value, (float, int)): | 409 if not isinstance(value, (float, int)): |
| 485 raise errors.MonitoringInvalidValueTypeError(self._name, value) | 410 raise errors.MonitoringInvalidValueTypeError(self._name, value) |
| 486 self._set(fields, target_fields, float(value), enforce_ge=True) | 411 self._set(fields, target_fields, float(value), enforce_ge=True) |
| 487 | 412 |
| 488 def is_cumulative(self): | 413 def is_cumulative(self): |
| 489 return True | 414 return True |
| 490 | 415 |
| 491 | 416 |
| 492 class FloatMetric(NumericMetric): | 417 class FloatMetric(NumericMetric): |
| 493 """A metric whose value type is a float.""" | 418 """A metric whose value type is a float.""" |
| 494 | 419 |
| 495 def _populate_value(self, metric, value, start_time): | 420 def _populate_value(self, metric, value): |
| 496 metric.noncumulative_double_value = value | |
| 497 | |
| 498 def _populate_value_new(self, metric, value): | |
| 499 metric.double_value = value | 421 metric.double_value = value |
| 500 | 422 |
| 501 def _populate_value_type(self, data_set_pb): | 423 def _populate_value_type(self, data_set_pb): |
| 502 data_set_pb.value_type = new_metrics_pb2.DOUBLE | 424 data_set_pb.value_type = metrics_pb2.DOUBLE |
| 503 | 425 |
| 504 def set(self, value, fields=None, target_fields=None): | 426 def set(self, value, fields=None, target_fields=None): |
| 505 if not isinstance(value, (float, int)): | 427 if not isinstance(value, (float, int)): |
| 506 raise errors.MonitoringInvalidValueTypeError(self._name, value) | 428 raise errors.MonitoringInvalidValueTypeError(self._name, value) |
| 507 self._set(fields, target_fields, float(value)) | 429 self._set(fields, target_fields, float(value)) |
| 508 | 430 |
| 509 def is_cumulative(self): | 431 def is_cumulative(self): |
| 510 return False | 432 return False |
| 511 | 433 |
| 512 | 434 |
| 513 class _DistributionMetricBase(Metric): | 435 class _DistributionMetricBase(Metric): |
| 514 """A metric that holds a distribution of values. | 436 """A metric that holds a distribution of values. |
| 515 | 437 |
| 516 By default buckets are chosen from a geometric progression, each bucket being | 438 By default buckets are chosen from a geometric progression, each bucket being |
| 517 approximately 1.59 times bigger than the last. In practice this is suitable | 439 approximately 1.59 times bigger than the last. In practice this is suitable |
| 518 for many kinds of data, but you may want to provide a FixedWidthBucketer or | 440 for many kinds of data, but you may want to provide a FixedWidthBucketer or |
| 519 GeometricBucketer with different parameters.""" | 441 GeometricBucketer with different parameters.""" |
| 520 | 442 |
| 521 CANONICAL_SPEC_TYPES = { | 443 def __init__(self, name, description, field_spec, is_cumulative=True, |
| 522 2: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_2, | |
| 523 10**0.2: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10_P_0_2, | |
| 524 10: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10, | |
| 525 } | |
| 526 | |
| 527 def __init__(self, name, description, field_spec=None, is_cumulative=True, | |
| 528 bucketer=None, start_time=None, units=None): | 444 bucketer=None, start_time=None, units=None): |
| 529 super(_DistributionMetricBase, self).__init__( | 445 super(_DistributionMetricBase, self).__init__( |
| 530 name, description, field_spec, units=units) | 446 name, description, field_spec, units=units) |
| 531 self._start_time = start_time | 447 self._start_time = start_time |
| 532 | 448 |
| 533 if bucketer is None: | 449 if bucketer is None: |
| 534 bucketer = distribution.GeometricBucketer() | 450 bucketer = distribution.GeometricBucketer() |
| 535 | 451 |
| 536 self._is_cumulative = is_cumulative | 452 self._is_cumulative = is_cumulative |
| 537 self.bucketer = bucketer | 453 self.bucketer = bucketer |
| 538 | 454 |
| 539 def _populate_value(self, metric, value, start_time): | 455 def _populate_value(self, metric, value): |
| 540 pb = metric.distribution | |
| 541 | |
| 542 pb.is_cumulative = self._is_cumulative | |
| 543 if self._is_cumulative: | |
| 544 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND) | |
| 545 | |
| 546 # Copy the bucketer params. | |
| 547 if (value.bucketer.width == 0 and | |
| 548 value.bucketer.growth_factor in self.CANONICAL_SPEC_TYPES): | |
| 549 pb.spec_type = self.CANONICAL_SPEC_TYPES[value.bucketer.growth_factor] | |
| 550 else: | |
| 551 pb.spec_type = metrics_pb2.PrecomputedDistribution.CUSTOM_PARAMETERIZED | |
| 552 pb.width = value.bucketer.width | |
| 553 pb.growth_factor = value.bucketer.growth_factor | |
| 554 pb.num_buckets = value.bucketer.num_finite_buckets | |
| 555 | |
| 556 # Copy the distribution bucket values. Only include the finite buckets, not | |
| 557 # the overflow buckets on each end. | |
| 558 pb.bucket.extend(self._running_zero_generator( | |
| 559 value.buckets.get(i, 0) for i in | |
| 560 xrange(1, value.bucketer.total_buckets - 1))) | |
| 561 | |
| 562 # Add the overflow buckets if present. | |
| 563 if value.bucketer.underflow_bucket in value.buckets: | |
| 564 pb.underflow = value.buckets[value.bucketer.underflow_bucket] | |
| 565 if value.bucketer.overflow_bucket in value.buckets: | |
| 566 pb.overflow = value.buckets[value.bucketer.overflow_bucket] | |
| 567 | |
| 568 if value.count != 0: | |
| 569 pb.mean = float(value.sum) / value.count | |
| 570 | |
| 571 def _populate_value_new(self, metric, value): | |
| 572 pb = metric.distribution_value | 456 pb = metric.distribution_value |
| 573 | 457 |
| 574 # Copy the bucketer params. | 458 # Copy the bucketer params. |
| 575 if value.bucketer.width == 0: | 459 if value.bucketer.width == 0: |
| 576 pb.exponential_buckets.growth_factor = value.bucketer.growth_factor | 460 pb.exponential_buckets.growth_factor = value.bucketer.growth_factor |
| 577 pb.exponential_buckets.scale = 1.0 | 461 pb.exponential_buckets.scale = value.bucketer.scale |
| 578 pb.exponential_buckets.num_finite_buckets = ( | 462 pb.exponential_buckets.num_finite_buckets = ( |
| 579 value.bucketer.num_finite_buckets) | 463 value.bucketer.num_finite_buckets) |
| 580 else: | 464 else: |
| 581 pb.linear_buckets.width = value.bucketer.width | 465 pb.linear_buckets.width = value.bucketer.width |
| 582 pb.linear_buckets.offset = 0.0 | 466 pb.linear_buckets.offset = 0.0 |
| 583 pb.linear_buckets.num_finite_buckets = value.bucketer.num_finite_buckets | 467 pb.linear_buckets.num_finite_buckets = value.bucketer.num_finite_buckets |
| 584 | 468 |
| 585 # Copy the distribution bucket values. Include the overflow buckets on | 469 # Copy the distribution bucket values. Include the overflow buckets on |
| 586 # either end. | 470 # either end. |
| 587 pb.bucket_count.extend( | 471 pb.bucket_count.extend( |
| 588 value.buckets.get(i, 0) for i in | 472 value.buckets.get(i, 0) for i in |
| 589 xrange(0, value.bucketer.total_buckets)) | 473 xrange(0, value.bucketer.total_buckets)) |
| 590 | 474 |
| 591 pb.count = value.count | 475 pb.count = value.count |
| 592 pb.mean = float(value.sum) / max(value.count, 1) | 476 pb.mean = float(value.sum) / max(value.count, 1) |
| 593 | 477 |
| 594 def _populate_value_type(self, data_set_pb): | 478 def _populate_value_type(self, data_set_pb): |
| 595 data_set_pb.value_type = new_metrics_pb2.DISTRIBUTION | 479 data_set_pb.value_type = metrics_pb2.DISTRIBUTION |
| 596 | |
| 597 @staticmethod | |
| 598 def _running_zero_generator(iterable): | |
| 599 """Compresses sequences of zeroes in the iterable into negative zero counts. | |
| 600 | |
| 601 For example an input of [1, 0, 0, 0, 2] is converted to [1, -3, 2]. | |
| 602 """ | |
| 603 | |
| 604 count = 0 | |
| 605 | |
| 606 for value in iterable: | |
| 607 if value == 0: | |
| 608 count += 1 | |
| 609 else: | |
| 610 if count != 0: | |
| 611 yield -count | |
| 612 count = 0 | |
| 613 yield value | |
| 614 | 480 |
| 615 def add(self, value, fields=None, target_fields=None): | 481 def add(self, value, fields=None, target_fields=None): |
| 616 def modify_fn(dist, value): | 482 def modify_fn(dist, value): |
| 617 if dist == 0: | 483 if dist == 0: |
| 618 dist = distribution.Distribution(self.bucketer) | 484 dist = distribution.Distribution(self.bucketer) |
| 619 dist.add(value) | 485 dist.add(value) |
| 620 return dist | 486 return dist |
| 621 | 487 |
| 622 self._incr(fields, target_fields, value, modify_fn=modify_fn) | 488 self._incr(fields, target_fields, value, modify_fn=modify_fn) |
| 623 | 489 |
| (...skipping 15 matching lines...) Expand all Loading... |
| 639 | 505 |
| 640 self._set(fields, target_fields, value) | 506 self._set(fields, target_fields, value) |
| 641 | 507 |
| 642 def is_cumulative(self): | 508 def is_cumulative(self): |
| 643 return self._is_cumulative | 509 return self._is_cumulative |
| 644 | 510 |
| 645 | 511 |
| 646 class CumulativeDistributionMetric(_DistributionMetricBase): | 512 class CumulativeDistributionMetric(_DistributionMetricBase): |
| 647 """A DistributionMetric with is_cumulative set to True.""" | 513 """A DistributionMetric with is_cumulative set to True.""" |
| 648 | 514 |
| 649 def __init__(self, name, description, field_spec=None, bucketer=None, | 515 def __init__(self, name, description, field_spec, bucketer=None, units=None): |
| 650 units=None): | |
| 651 super(CumulativeDistributionMetric, self).__init__( | 516 super(CumulativeDistributionMetric, self).__init__( |
| 652 name, description, field_spec, | 517 name, description, field_spec, |
| 653 is_cumulative=True, | 518 is_cumulative=True, |
| 654 bucketer=bucketer, | 519 bucketer=bucketer, |
| 655 units=units) | 520 units=units) |
| 656 | 521 |
| 657 | 522 |
| 658 class NonCumulativeDistributionMetric(_DistributionMetricBase): | 523 class NonCumulativeDistributionMetric(_DistributionMetricBase): |
| 659 """A DistributionMetric with is_cumulative set to False.""" | 524 """A DistributionMetric with is_cumulative set to False.""" |
| 660 | 525 |
| 661 def __init__(self, name, description, field_spec=None, bucketer=None, | 526 def __init__(self, name, description, field_spec, bucketer=None, units=None): |
| 662 units=None): | |
| 663 super(NonCumulativeDistributionMetric, self).__init__( | 527 super(NonCumulativeDistributionMetric, self).__init__( |
| 664 name, description, field_spec, | 528 name, description, field_spec, |
| 665 is_cumulative=False, | 529 is_cumulative=False, |
| 666 bucketer=bucketer, | 530 bucketer=bucketer, |
| 667 units=units) | 531 units=units) |
| 668 | 532 |
| 669 | 533 |
| 670 class MetaMetricsDataUnits(type): | 534 class MetricsDataUnits(object): |
| 671 """Metaclass to populate the enum values of metrics_pb2.MetricsData.Units.""" | 535 """An container for units of measurement for Metrics data.""" |
| 672 def __new__(mcs, name, bases, attrs): | |
| 673 attrs.update(metrics_pb2.MetricsData.Units.items()) | |
| 674 return super(MetaMetricsDataUnits, mcs).__new__(mcs, name, bases, attrs) | |
| 675 | 536 |
| 676 | 537 UNKNOWN_UNITS = '{unknown}' |
| 677 class MetricsDataUnits(object): | 538 SECONDS = 's' |
| 678 """An enumeration class for units of measurement for Metrics data. | 539 MILLISECONDS = 'ms' |
| 679 See infra_libs/ts_mon/protos/metrics.proto for a full list of supported units. | 540 MICROSECONDS = 'us' |
| 680 """ | 541 NANOSECONDS = 'ns' |
| 681 __metaclass__ = MetaMetricsDataUnits | 542 BITS = 'B' |
| 682 | 543 BYTES = 'By' |
| 683 _UNITS_TO_STRING = { | 544 KILOBYTES = 'kBy' |
| 684 MetricsDataUnits.UNKNOWN_UNITS: '{unknown}', | 545 MEGABYTES = 'MBy' |
| 685 MetricsDataUnits.SECONDS: 's', | 546 GIGABYTES = 'GBy' |
| 686 MetricsDataUnits.MILLISECONDS: 'ms', | 547 KIBIBYTES = 'kiBy' |
| 687 MetricsDataUnits.MICROSECONDS: 'us', | 548 MEBIBYTES = 'MiBy' |
| 688 MetricsDataUnits.NANOSECONDS: 'ns', | 549 GIBIBYTES = 'GiBy' |
| 689 MetricsDataUnits.BITS: 'B', | 550 AMPS = 'A' |
| 690 MetricsDataUnits.BYTES: 'By', | 551 MILLIAMPS = 'mA' |
| 691 MetricsDataUnits.KILOBYTES: 'kBy', | 552 DEGREES_CELSIUS = 'Cel' |
| 692 MetricsDataUnits.MEGABYTES: 'MBy', | |
| 693 MetricsDataUnits.GIGABYTES: 'GBy', | |
| 694 MetricsDataUnits.KIBIBYTES: 'kiBy', | |
| 695 MetricsDataUnits.MEBIBYTES: 'MiBy', | |
| 696 MetricsDataUnits.GIBIBYTES: 'GiBy', | |
| 697 MetricsDataUnits.AMPS: 'A', | |
| 698 MetricsDataUnits.MILLIAMPS : 'mA', | |
| 699 MetricsDataUnits.DEGREES_CELSIUS: 'Cel' | |
| 700 } | |
| OLD | NEW |