Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(35)

Side by Side Diff: client/third_party/infra_libs/ts_mon/common/metrics.py

Issue 2991803002: Update infra_libs to 1.1.15 / 0b44aba87c1c6538439df6d24a409870810747ab (Closed)
Patch Set: fix Created 3 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2015 The Chromium Authors. All rights reserved. 1 # Copyright 2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Classes representing individual metrics that can be sent.""" 5 """Classes representing individual metrics that can be sent."""
6 6
7 import copy 7 import copy
8 import re 8 import re
9 9
10 from infra_libs.ts_mon.protos.current import metrics_pb2 10 from infra_libs.ts_mon.protos import metrics_pb2
11 from infra_libs.ts_mon.protos.new import metrics_pb2 as new_metrics_pb2
12 11
13 from infra_libs.ts_mon.common import distribution 12 from infra_libs.ts_mon.common import distribution
14 from infra_libs.ts_mon.common import errors 13 from infra_libs.ts_mon.common import errors
15 from infra_libs.ts_mon.common import interface 14 from infra_libs.ts_mon.common import interface
16 15
17 16
18 MICROSECONDS_PER_SECOND = 1000000 17 MICROSECONDS_PER_SECOND = 1000000
19 18
20 19
21 class Field(object): 20 class Field(object):
22 FIELD_NAME_PATTERN = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') 21 FIELD_NAME_PATTERN = re.compile(r'[A-Za-z_][A-Za-z0-9_]*')
23 22
24 allowed_python_types = None 23 allowed_python_types = None
25 v1_type = None 24 type_enum = None
26 v2_type = None 25 field_name = None
27 v1_field = None
28 v2_field = None
29 26
30 def __init__(self, name): 27 def __init__(self, name):
31 if not self.FIELD_NAME_PATTERN.match(name): 28 if not self.FIELD_NAME_PATTERN.match(name):
32 raise errors.MetricDefinitionError( 29 raise errors.MetricDefinitionError(
33 'Invalid metric field name "%s" - must match the regex "%s"' % ( 30 'Invalid metric field name "%s" - must match the regex "%s"' % (
34 name, self.FIELD_NAME_PATTERN.pattern)) 31 name, self.FIELD_NAME_PATTERN.pattern))
35 32
36 self.name = name 33 self.name = name
37 34
38 def validate_value(self, metric_name, value): 35 def validate_value(self, metric_name, value):
39 if not isinstance(value, self.allowed_python_types): 36 if not isinstance(value, self.allowed_python_types):
40 raise errors.MonitoringInvalidFieldTypeError( 37 raise errors.MonitoringInvalidFieldTypeError(
41 metric_name, self.name, value) 38 metric_name, self.name, value)
42 39
43 def populate_proto_v1(self, proto, value): 40 def populate_proto(self, proto, value):
44 setattr(proto, self.v1_field, value) 41 setattr(proto, self.field_name, value)
45
46 def populate_proto_v2(self, proto, value):
47 setattr(proto, self.v2_field, value)
48 42
49 43
50 class StringField(Field): 44 class StringField(Field):
51 allowed_python_types = basestring 45 allowed_python_types = basestring
52 v1_type = metrics_pb2.MetricsField.STRING 46 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.STRING
53 v2_type = new_metrics_pb2.MetricsDataSet.MetricFieldDescriptor.STRING 47 field_name = 'string_value'
54 v1_field = 'string_value'
55 v2_field = 'string_value'
56 48
57 49
58 class IntegerField(Field): 50 class IntegerField(Field):
59 allowed_python_types = (int, long) 51 allowed_python_types = (int, long)
60 v1_type = metrics_pb2.MetricsField.INT 52 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.INT64
61 v2_type = new_metrics_pb2.MetricsDataSet.MetricFieldDescriptor.INT64 53 field_name = 'int64_value'
62 v1_field = 'int_value'
63 v2_field = 'int64_value'
64 54
65 55
66 class BooleanField(Field): 56 class BooleanField(Field):
67 allowed_python_types = bool 57 allowed_python_types = bool
68 v1_type = metrics_pb2.MetricsField.BOOL 58 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.BOOL
69 v2_type = new_metrics_pb2.MetricsDataSet.MetricFieldDescriptor.BOOL 59 field_name = 'bool_value'
70 v1_field = 'bool_value'
71 v2_field = 'bool_value'
72 60
73 61
74 class Metric(object): 62 class Metric(object):
75 """Abstract base class for a metric. 63 """Abstract base class for a metric.
76 64
77 A Metric is an attribute that may be monitored across many targets. Examples 65 A Metric is an attribute that may be monitored across many targets. Examples
78 include disk usage or the number of requests a server has received. A single 66 include disk usage or the number of requests a server has received. A single
79 process may keep track of many metrics. 67 process may keep track of many metrics.
80 68
81 Note that Metric objects may be initialized at any time (for example, at the 69 Note that Metric objects may be initialized at any time (for example, at the
82 top of a library), but cannot be sent until the underlying Monitor object 70 top of a library), but cannot be sent until the underlying Monitor object
83 has been set up (usually by the top-level process parsing the command line). 71 has been set up (usually by the top-level process parsing the command line).
84 72
85 A Metric can actually store multiple values that are identified by a set of 73 A Metric can actually store multiple values that are identified by a set of
86 fields (which are themselves key-value pairs). Fields can be passed to the 74 fields (which are themselves key-value pairs). Fields can be passed to the
87 set() or increment() methods to modify a particular value, or passed to the 75 set() or increment() methods to modify a particular value, or passed to the
88 constructor in which case they will be used as the defaults for this Metric. 76 constructor in which case they will be used as the defaults for this Metric.
89 77
90 The unit of measurement for Metric data can be specified with MetricsDataUnits 78 The unit of measurement for Metric data should be specified with
91 when a Metric object is created: 79 MetricsDataUnits when a Metric object is created:
92 e.g., MetricsDataUnits.SECONDS, MetricsDataUnits.BYTES, and etc.., 80 e.g., MetricsDataUnits.SECONDS, MetricsDataUnits.BYTES, and etc..,
93 A full list of supported units can be found in the following protobuf file 81 See `MetricsDataUnits` class for a full list of units.
94 : infra_libs/ts_mon/protos/metrics.proto
95 82
96 Do not directly instantiate an object of this class. 83 Do not directly instantiate an object of this class.
97 Use the concrete child classes instead: 84 Use the concrete child classes instead:
98 * StringMetric for metrics with string value 85 * StringMetric for metrics with string value
99 * BooleanMetric for metrics with boolean values 86 * BooleanMetric for metrics with boolean values
100 * CounterMetric for metrics with monotonically increasing integer values 87 * CounterMetric for metrics with monotonically increasing integer values
101 * GaugeMetric for metrics with arbitrarily varying integer values 88 * GaugeMetric for metrics with arbitrarily varying integer values
102 * CumulativeMetric for metrics with monotonically increasing float values 89 * CumulativeMetric for metrics with monotonically increasing float values
103 * FloatMetric for metrics with arbitrarily varying float values 90 * FloatMetric for metrics with arbitrarily varying float values
104 91
105 See http://go/inframon-doc for help designing and using your metrics. 92 See http://go/inframon-doc for help designing and using your metrics.
106 """ 93 """
107 94
108 def __init__(self, name, description, field_spec, units=None): 95 def __init__(self, name, description, field_spec, units=None):
109 """Create an instance of a Metric. 96 """Create an instance of a Metric.
110 97
111 Args: 98 Args:
112 name (str): the file-like name of this metric 99 name (str): the file-like name of this metric
113 description (string): help string for the metric. Should be enough to 100 description (string): help string for the metric. Should be enough to
114 know what the metric is about. 101 know what the metric is about.
115 field_spec (list): a list of Field subclasses to define the fields that 102 field_spec (list): a list of Field subclasses to define the fields that
116 are allowed on this metric. Pass a list of either 103 are allowed on this metric. Pass a list of either
117 StringField, IntegerField or BooleanField here. 104 StringField, IntegerField or BooleanField here.
118 units (int): the unit used to measure data for given 105 units (string): the unit used to measure data for given metric. Some
119 metric. Please use the attributes of MetricDataUnit to find 106 common units are pre-defined in the MetricsDataUnits
120 valid integer values for this argument. 107 class.
121 """ 108 """
122 field_spec = field_spec or [] 109 field_spec = field_spec or []
123 110
124 self._name = name.lstrip('/') 111 self._name = name.lstrip('/')
125 112
126 if not isinstance(description, basestring): 113 if not isinstance(description, basestring):
127 raise errors.MetricDefinitionError('Metric description must be a string') 114 raise errors.MetricDefinitionError('Metric description must be a string')
128 if not description: 115 if not description:
129 raise errors.MetricDefinitionError('Metric must have a description') 116 raise errors.MetricDefinitionError('Metric must have a description')
130 if (not isinstance(field_spec, (list, tuple)) or 117 if (not isinstance(field_spec, (list, tuple)) or
131 any(not isinstance(x, Field) for x in field_spec)): 118 any(not isinstance(x, Field) for x in field_spec)):
132 raise errors.MetricDefinitionError( 119 raise errors.MetricDefinitionError(
133 'Metric constructor takes a list of Fields, or None') 120 'Metric constructor takes a list of Fields, or None')
134 if len(field_spec) > 7: 121 if len(field_spec) > 7:
135 raise errors.MonitoringTooManyFieldsError(self._name, field_spec) 122 raise errors.MonitoringTooManyFieldsError(self._name, field_spec)
136 123
137 self._start_time = None 124 self._start_time = None
138 self._field_spec = field_spec 125 self._field_spec = field_spec
139 self._sorted_field_names = sorted(x.name for x in field_spec) 126 self._sorted_field_names = sorted(x.name for x in field_spec)
140 self._description = description 127 self._description = description
141 self._units = units 128 self._units = units
142 129
143 interface.register(self) 130 interface.register(self)
144 131
145 @property 132 @property
133 def field_spec(self):
134 return list(self._field_spec)
135
136 @property
146 def name(self): 137 def name(self):
147 return self._name 138 return self._name
148 139
149 @property 140 @property
150 def start_time(self): 141 def start_time(self):
151 return self._start_time 142 return self._start_time
152 143
144 @property
145 def units(self):
146 return self._units
147
153 def is_cumulative(self): 148 def is_cumulative(self):
154 raise NotImplementedError() 149 raise NotImplementedError()
155 150
156 def unregister(self): 151 def unregister(self):
157 interface.unregister(self) 152 interface.unregister(self)
158 153
159 @staticmethod 154 def populate_data_set(self, data_set):
160 def _map_units_to_string(units):
161 """Map MetricsDataUnits to the corresponding string according to:
162 http://unitsofmeasure.org/ucum.html because that's what the new proto
163 requires."""
164 if units in _UNITS_TO_STRING:
165 return _UNITS_TO_STRING[units]
166 else:
167 return '{unknown}'
168
169 def _populate_data_set(self, data_set):
170 """Populate MetricsDataSet.""" 155 """Populate MetricsDataSet."""
171 data_set.metric_name = '%s%s' % (interface.state.metric_name_prefix, 156 data_set.metric_name = '%s%s' % (interface.state.metric_name_prefix,
172 self._name) 157 self._name)
173 data_set.description = self._description or '' 158 data_set.description = self._description or ''
174 data_set.annotations.unit = self._map_units_to_string(self._units) 159 if self._units is not None:
160 data_set.annotations.unit = self._units
175 161
176 if self.is_cumulative(): 162 if self.is_cumulative():
177 data_set.stream_kind = new_metrics_pb2.CUMULATIVE 163 data_set.stream_kind = metrics_pb2.CUMULATIVE
178 else: 164 else:
179 data_set.stream_kind = new_metrics_pb2.GAUGE 165 data_set.stream_kind = metrics_pb2.GAUGE
180 166
181 self._populate_value_type(data_set) 167 self._populate_value_type(data_set)
182 self._populate_field_descriptors(data_set) 168 self._populate_field_descriptors(data_set)
183 169
184 def _populate_data(self, data, start_time, end_time, fields, value): 170 def populate_data(self, data, start_time, end_time, fields, value):
185 """Populate a new metrics_pb2.MetricsData. 171 """Populate a new metrics_pb2.MetricsData.
186 172
187 Args: 173 Args:
188 data_ (new_metrics_pb2.MetricsData): protocol buffer into 174 data (metrics_pb2.MetricsData): protocol buffer into
189 which to populate the current metric values. 175 which to populate the current metric values.
190 start_time (int): timestamp in microseconds since UNIX epoch. 176 start_time (int): timestamp in microseconds since UNIX epoch.
191 """ 177 """
192 data.start_timestamp.seconds = int(start_time) 178 data.start_timestamp.seconds = int(start_time)
193 data.end_timestamp.seconds = int(end_time) 179 data.end_timestamp.seconds = int(end_time)
194 180
195 self._populate_fields_new(data, fields) 181 self._populate_fields(data, fields)
196 self._populate_value_new(data, value) 182 self._populate_value(data, value)
197
198 def serialize_to(self, metric_pb, start_time, fields, value, target):
199 """Generate metrics_pb2.MetricsData messages for this metric.
200
201 Args:
202 metric_pb (metrics_pb2.MetricsData): protocol buffer into which
203 to serialize the current metric values.
204 start_time (int): timestamp in microseconds since UNIX epoch.
205 target (Target): a Target to use.
206 """
207
208 metric_pb.metric_name_prefix = interface.state.metric_name_prefix
209 metric_pb.name = self._name
210 metric_pb.description = self._description
211 if self._units is not None:
212 metric_pb.units = self._units
213
214 self._populate_value(metric_pb, value, start_time)
215 self._populate_fields(metric_pb, fields)
216
217 target._populate_target_pb(metric_pb)
218 183
219 def _populate_field_descriptors(self, data_set): 184 def _populate_field_descriptors(self, data_set):
220 """Populate `field_descriptor` in MetricsDataSet. 185 """Populate `field_descriptor` in MetricsDataSet.
221 186
222 Args: 187 Args:
223 data_set (new_metrics_pb2.MetricsDataSet): a data set protobuf to populate 188 data_set (metrics_pb2.MetricsDataSet): a data set protobuf to populate
224 """ 189 """
225 for spec in self._field_spec: 190 for spec in self._field_spec:
226 descriptor = data_set.field_descriptor.add() 191 descriptor = data_set.field_descriptor.add()
227 descriptor.name = spec.name 192 descriptor.name = spec.name
228 descriptor.field_type = spec.v2_type 193 descriptor.field_type = spec.type_enum
229 194
230 def _populate_fields_new(self, data, field_values): 195 def _populate_fields(self, data, field_values):
231 """Fill in the fields attribute of a metric protocol buffer. 196 """Fill in the fields attribute of a metric protocol buffer.
232 197
233 Args: 198 Args:
234 metric (metrics_pb2.MetricsData): a metrics protobuf to populate 199 metric (metrics_pb2.MetricsData): a metrics protobuf to populate
235 field_values (tuple): field values 200 field_values (tuple): field values
236 """ 201 """
237 for spec, value in zip(self._field_spec, field_values): 202 for spec, value in zip(self._field_spec, field_values):
238 field = data.field.add() 203 field = data.field.add()
239 field.name = spec.name 204 field.name = spec.name
240 spec.populate_proto_v2(field, value) 205 spec.populate_proto(field, value)
241
242 def _populate_fields(self, metric, field_values):
243 """Fill in the fields attribute of a metric protocol buffer.
244
245 Args:
246 metric (metrics_pb2.MetricsData): a metrics protobuf to populate
247 field_values (tuple): field values
248 """
249 for spec, value in zip(self._field_spec, field_values):
250 field = metric.fields.add()
251 field.name = spec.name
252 field.type = spec.v1_type
253 spec.populate_proto_v1(field, value)
254 206
255 def _validate_fields(self, fields): 207 def _validate_fields(self, fields):
256 """Checks the correct number and types of field values were provided. 208 """Checks the correct number and types of field values were provided.
257 209
258 Args: 210 Args:
259 fields (dict): A dict of field values given by the user, or None. 211 fields (dict): A dict of field values given by the user, or None.
260 212
261 Returns: 213 Returns:
262 fields' values as a tuple, in the same order as the field_spec. 214 fields' values as a tuple, in the same order as the field_spec.
263 215
(...skipping 11 matching lines...) Expand all
275 227
276 if sorted(fields) != self._sorted_field_names: 228 if sorted(fields) != self._sorted_field_names:
277 raise errors.WrongFieldsError( 229 raise errors.WrongFieldsError(
278 self.name, fields.keys(), self._sorted_field_names) 230 self.name, fields.keys(), self._sorted_field_names)
279 231
280 for spec in self._field_spec: 232 for spec in self._field_spec:
281 spec.validate_value(self.name, fields[spec.name]) 233 spec.validate_value(self.name, fields[spec.name])
282 234
283 return tuple(fields[spec.name] for spec in self._field_spec) 235 return tuple(fields[spec.name] for spec in self._field_spec)
284 236
285 def _populate_value(self, metric, value, start_time): 237 def _populate_value(self, data, value):
286 """Fill in the the data values of a metric protocol buffer. 238 """Fill in the the data values of a metric protocol buffer.
287 239
288 Args: 240 Args:
289 metric (metrics_pb2.MetricsData): a metrics protobuf to populate
290 value (see concrete class): the value of the metric to be set
291 start_time (int): timestamp in microseconds since UNIX epoch.
292 """
293 raise NotImplementedError()
294
295 def _populate_value_new(self, data, value):
296 """Fill in the the data values of a metric protocol buffer.
297
298 Args:
299 data (metrics_pb2.MetricsData): a metrics protobuf to populate 241 data (metrics_pb2.MetricsData): a metrics protobuf to populate
300 value (see concrete class): the value of the metric to be set 242 value (see concrete class): the value of the metric to be set
301 """ 243 """
302 raise NotImplementedError() 244 raise NotImplementedError()
303 245
304 def _populate_value_type(self, data_set): 246 def _populate_value_type(self, data_set):
305 """Fill in the the data values of a metric protocol buffer. 247 """Fill in the the data values of a metric protocol buffer.
306 248
307 Args: 249 Args:
308 data_set (metrics_pb2.MetricsDataSet): a MetricsDataSet protobuf to 250 data_set (metrics_pb2.MetricsDataSet): a MetricsDataSet protobuf to
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
351 293
352 def _incr(self, fields, target_fields, delta, modify_fn=None): 294 def _incr(self, fields, target_fields, delta, modify_fn=None):
353 interface.state.store.incr( 295 interface.state.store.incr(
354 self.name, self._validate_fields(fields), target_fields, 296 self.name, self._validate_fields(fields), target_fields,
355 delta, modify_fn=modify_fn) 297 delta, modify_fn=modify_fn)
356 298
357 299
358 class StringMetric(Metric): 300 class StringMetric(Metric):
359 """A metric whose value type is a string.""" 301 """A metric whose value type is a string."""
360 302
361 def _populate_value(self, metric, value, start_time): 303 def _populate_value(self, data, value):
362 metric.string_value = value
363
364 def _populate_value_new(self, data, value):
365 data.string_value = value 304 data.string_value = value
366 305
367 def _populate_value_type(self, data_set): 306 def _populate_value_type(self, data_set):
368 data_set.value_type = new_metrics_pb2.STRING 307 data_set.value_type = metrics_pb2.STRING
369 308
370 def set(self, value, fields=None, target_fields=None): 309 def set(self, value, fields=None, target_fields=None):
371 if not isinstance(value, basestring): 310 if not isinstance(value, basestring):
372 raise errors.MonitoringInvalidValueTypeError(self._name, value) 311 raise errors.MonitoringInvalidValueTypeError(self._name, value)
373 self._set(fields, target_fields, value) 312 self._set(fields, target_fields, value)
374 313
375 def is_cumulative(self): 314 def is_cumulative(self):
376 return False 315 return False
377 316
378 317
379 class BooleanMetric(Metric): 318 class BooleanMetric(Metric):
380 """A metric whose value type is a boolean.""" 319 """A metric whose value type is a boolean."""
381 320
382 def _populate_value(self, metric, value, start_time): 321 def _populate_value(self, data, value):
383 metric.boolean_value = value
384
385 def _populate_value_new(self, data, value):
386 data.bool_value = value 322 data.bool_value = value
387 323
388 def _populate_value_type(self, data_set): 324 def _populate_value_type(self, data_set):
389 data_set.value_type = new_metrics_pb2.BOOL 325 data_set.value_type = metrics_pb2.BOOL
390 326
391 def set(self, value, fields=None, target_fields=None): 327 def set(self, value, fields=None, target_fields=None):
392 if not isinstance(value, bool): 328 if not isinstance(value, bool):
393 raise errors.MonitoringInvalidValueTypeError(self._name, value) 329 raise errors.MonitoringInvalidValueTypeError(self._name, value)
394 self._set(fields, target_fields, value) 330 self._set(fields, target_fields, value)
395 331
396 def is_cumulative(self): 332 def is_cumulative(self):
397 return False 333 return False
398 334
399 335
400 class NumericMetric(Metric): # pylint: disable=abstract-method 336 class NumericMetric(Metric): # pylint: disable=abstract-method
401 """Abstract base class for numeric (int or float) metrics.""" 337 """Abstract base class for numeric (int or float) metrics."""
402 338
403 def increment(self, fields=None, target_fields=None): 339 def increment(self, fields=None, target_fields=None):
404 self._incr(fields, target_fields, 1) 340 self._incr(fields, target_fields, 1)
405 341
406 def increment_by(self, step, fields=None, target_fields=None): 342 def increment_by(self, step, fields=None, target_fields=None):
407 self._incr(fields, target_fields, step) 343 self._incr(fields, target_fields, step)
408 344
409 345
410 class CounterMetric(NumericMetric): 346 class CounterMetric(NumericMetric):
411 """A metric whose value type is a monotonically increasing integer.""" 347 """A metric whose value type is a monotonically increasing integer."""
412 348
413 def __init__(self, name, description, field_spec=None, start_time=None, 349 def __init__(self, name, description, field_spec, start_time=None,
414 units=None): 350 units=None):
415 super(CounterMetric, self).__init__( 351 super(CounterMetric, self).__init__(
416 name, description, field_spec, units=units) 352 name, description, field_spec, units=units)
417 self._start_time = start_time 353 self._start_time = start_time
418 354
419 def _populate_value(self, metric, value, start_time): 355 def _populate_value(self, data, value):
420 metric.counter = value
421 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND)
422
423 def _populate_value_new(self, data, value):
424 data.int64_value = value 356 data.int64_value = value
425 357
426 def _populate_value_type(self, data_set): 358 def _populate_value_type(self, data_set):
427 data_set.value_type = new_metrics_pb2.INT64 359 data_set.value_type = metrics_pb2.INT64
428 360
429 def set(self, value, fields=None, target_fields=None): 361 def set(self, value, fields=None, target_fields=None):
430 if not isinstance(value, (int, long)): 362 if not isinstance(value, (int, long)):
431 raise errors.MonitoringInvalidValueTypeError(self._name, value) 363 raise errors.MonitoringInvalidValueTypeError(self._name, value)
432 self._set(fields, target_fields, value, enforce_ge=True) 364 self._set(fields, target_fields, value, enforce_ge=True)
433 365
434 def increment_by(self, step, fields=None, target_fields=None): 366 def increment_by(self, step, fields=None, target_fields=None):
435 if not isinstance(step, (int, long)): 367 if not isinstance(step, (int, long)):
436 raise errors.MonitoringInvalidValueTypeError(self._name, step) 368 raise errors.MonitoringInvalidValueTypeError(self._name, step)
437 self._incr(fields, target_fields, step) 369 self._incr(fields, target_fields, step)
438 370
439 def is_cumulative(self): 371 def is_cumulative(self):
440 return True 372 return True
441 373
442 374
443 class GaugeMetric(NumericMetric): 375 class GaugeMetric(NumericMetric):
444 """A metric whose value type is an integer.""" 376 """A metric whose value type is an integer."""
445 377
446 def _populate_value(self, metric, value, start_time): 378 def _populate_value(self, data, value):
447 metric.gauge = value
448
449 def _populate_value_new(self, data, value):
450 data.int64_value = value 379 data.int64_value = value
451 380
452 def _populate_value_type(self, data_set): 381 def _populate_value_type(self, data_set):
453 data_set.value_type = new_metrics_pb2.INT64 382 data_set.value_type = metrics_pb2.INT64
454 383
455 def set(self, value, fields=None, target_fields=None): 384 def set(self, value, fields=None, target_fields=None):
456 if not isinstance(value, (int, long)): 385 if not isinstance(value, (int, long)):
457 raise errors.MonitoringInvalidValueTypeError(self._name, value) 386 raise errors.MonitoringInvalidValueTypeError(self._name, value)
458 self._set(fields, target_fields, value) 387 self._set(fields, target_fields, value)
459 388
460 def is_cumulative(self): 389 def is_cumulative(self):
461 return False 390 return False
462 391
463 392
464 class CumulativeMetric(NumericMetric): 393 class CumulativeMetric(NumericMetric):
465 """A metric whose value type is a monotonically increasing float.""" 394 """A metric whose value type is a monotonically increasing float."""
466 395
467 def __init__(self, name, description, field_spec=None, start_time=None, 396 def __init__(self, name, description, field_spec, start_time=None,
468 units=None): 397 units=None):
469 super(CumulativeMetric, self).__init__( 398 super(CumulativeMetric, self).__init__(
470 name, description, field_spec, units=units) 399 name, description, field_spec, units=units)
471 self._start_time = start_time 400 self._start_time = start_time
472 401
473 def _populate_value(self, metric, value, start_time): 402 def _populate_value(self, data, value):
474 metric.cumulative_double_value = value
475 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND)
476
477 def _populate_value_new(self, data, value):
478 data.double_value = value 403 data.double_value = value
479 404
480 def _populate_value_type(self, data_set): 405 def _populate_value_type(self, data_set):
481 data_set.value_type = new_metrics_pb2.DOUBLE 406 data_set.value_type = metrics_pb2.DOUBLE
482 407
483 def set(self, value, fields=None, target_fields=None): 408 def set(self, value, fields=None, target_fields=None):
484 if not isinstance(value, (float, int)): 409 if not isinstance(value, (float, int)):
485 raise errors.MonitoringInvalidValueTypeError(self._name, value) 410 raise errors.MonitoringInvalidValueTypeError(self._name, value)
486 self._set(fields, target_fields, float(value), enforce_ge=True) 411 self._set(fields, target_fields, float(value), enforce_ge=True)
487 412
488 def is_cumulative(self): 413 def is_cumulative(self):
489 return True 414 return True
490 415
491 416
492 class FloatMetric(NumericMetric): 417 class FloatMetric(NumericMetric):
493 """A metric whose value type is a float.""" 418 """A metric whose value type is a float."""
494 419
495 def _populate_value(self, metric, value, start_time): 420 def _populate_value(self, metric, value):
496 metric.noncumulative_double_value = value
497
498 def _populate_value_new(self, metric, value):
499 metric.double_value = value 421 metric.double_value = value
500 422
501 def _populate_value_type(self, data_set_pb): 423 def _populate_value_type(self, data_set_pb):
502 data_set_pb.value_type = new_metrics_pb2.DOUBLE 424 data_set_pb.value_type = metrics_pb2.DOUBLE
503 425
504 def set(self, value, fields=None, target_fields=None): 426 def set(self, value, fields=None, target_fields=None):
505 if not isinstance(value, (float, int)): 427 if not isinstance(value, (float, int)):
506 raise errors.MonitoringInvalidValueTypeError(self._name, value) 428 raise errors.MonitoringInvalidValueTypeError(self._name, value)
507 self._set(fields, target_fields, float(value)) 429 self._set(fields, target_fields, float(value))
508 430
509 def is_cumulative(self): 431 def is_cumulative(self):
510 return False 432 return False
511 433
512 434
513 class _DistributionMetricBase(Metric): 435 class _DistributionMetricBase(Metric):
514 """A metric that holds a distribution of values. 436 """A metric that holds a distribution of values.
515 437
516 By default buckets are chosen from a geometric progression, each bucket being 438 By default buckets are chosen from a geometric progression, each bucket being
517 approximately 1.59 times bigger than the last. In practice this is suitable 439 approximately 1.59 times bigger than the last. In practice this is suitable
518 for many kinds of data, but you may want to provide a FixedWidthBucketer or 440 for many kinds of data, but you may want to provide a FixedWidthBucketer or
519 GeometricBucketer with different parameters.""" 441 GeometricBucketer with different parameters."""
520 442
521 CANONICAL_SPEC_TYPES = { 443 def __init__(self, name, description, field_spec, is_cumulative=True,
522 2: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_2,
523 10**0.2: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10_P_0_2,
524 10: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10,
525 }
526
527 def __init__(self, name, description, field_spec=None, is_cumulative=True,
528 bucketer=None, start_time=None, units=None): 444 bucketer=None, start_time=None, units=None):
529 super(_DistributionMetricBase, self).__init__( 445 super(_DistributionMetricBase, self).__init__(
530 name, description, field_spec, units=units) 446 name, description, field_spec, units=units)
531 self._start_time = start_time 447 self._start_time = start_time
532 448
533 if bucketer is None: 449 if bucketer is None:
534 bucketer = distribution.GeometricBucketer() 450 bucketer = distribution.GeometricBucketer()
535 451
536 self._is_cumulative = is_cumulative 452 self._is_cumulative = is_cumulative
537 self.bucketer = bucketer 453 self.bucketer = bucketer
538 454
539 def _populate_value(self, metric, value, start_time): 455 def _populate_value(self, metric, value):
540 pb = metric.distribution
541
542 pb.is_cumulative = self._is_cumulative
543 if self._is_cumulative:
544 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND)
545
546 # Copy the bucketer params.
547 if (value.bucketer.width == 0 and
548 value.bucketer.growth_factor in self.CANONICAL_SPEC_TYPES):
549 pb.spec_type = self.CANONICAL_SPEC_TYPES[value.bucketer.growth_factor]
550 else:
551 pb.spec_type = metrics_pb2.PrecomputedDistribution.CUSTOM_PARAMETERIZED
552 pb.width = value.bucketer.width
553 pb.growth_factor = value.bucketer.growth_factor
554 pb.num_buckets = value.bucketer.num_finite_buckets
555
556 # Copy the distribution bucket values. Only include the finite buckets, not
557 # the overflow buckets on each end.
558 pb.bucket.extend(self._running_zero_generator(
559 value.buckets.get(i, 0) for i in
560 xrange(1, value.bucketer.total_buckets - 1)))
561
562 # Add the overflow buckets if present.
563 if value.bucketer.underflow_bucket in value.buckets:
564 pb.underflow = value.buckets[value.bucketer.underflow_bucket]
565 if value.bucketer.overflow_bucket in value.buckets:
566 pb.overflow = value.buckets[value.bucketer.overflow_bucket]
567
568 if value.count != 0:
569 pb.mean = float(value.sum) / value.count
570
571 def _populate_value_new(self, metric, value):
572 pb = metric.distribution_value 456 pb = metric.distribution_value
573 457
574 # Copy the bucketer params. 458 # Copy the bucketer params.
575 if value.bucketer.width == 0: 459 if value.bucketer.width == 0:
576 pb.exponential_buckets.growth_factor = value.bucketer.growth_factor 460 pb.exponential_buckets.growth_factor = value.bucketer.growth_factor
577 pb.exponential_buckets.scale = 1.0 461 pb.exponential_buckets.scale = value.bucketer.scale
578 pb.exponential_buckets.num_finite_buckets = ( 462 pb.exponential_buckets.num_finite_buckets = (
579 value.bucketer.num_finite_buckets) 463 value.bucketer.num_finite_buckets)
580 else: 464 else:
581 pb.linear_buckets.width = value.bucketer.width 465 pb.linear_buckets.width = value.bucketer.width
582 pb.linear_buckets.offset = 0.0 466 pb.linear_buckets.offset = 0.0
583 pb.linear_buckets.num_finite_buckets = value.bucketer.num_finite_buckets 467 pb.linear_buckets.num_finite_buckets = value.bucketer.num_finite_buckets
584 468
585 # Copy the distribution bucket values. Include the overflow buckets on 469 # Copy the distribution bucket values. Include the overflow buckets on
586 # either end. 470 # either end.
587 pb.bucket_count.extend( 471 pb.bucket_count.extend(
588 value.buckets.get(i, 0) for i in 472 value.buckets.get(i, 0) for i in
589 xrange(0, value.bucketer.total_buckets)) 473 xrange(0, value.bucketer.total_buckets))
590 474
591 pb.count = value.count 475 pb.count = value.count
592 pb.mean = float(value.sum) / max(value.count, 1) 476 pb.mean = float(value.sum) / max(value.count, 1)
593 477
594 def _populate_value_type(self, data_set_pb): 478 def _populate_value_type(self, data_set_pb):
595 data_set_pb.value_type = new_metrics_pb2.DISTRIBUTION 479 data_set_pb.value_type = metrics_pb2.DISTRIBUTION
596
597 @staticmethod
598 def _running_zero_generator(iterable):
599 """Compresses sequences of zeroes in the iterable into negative zero counts.
600
601 For example an input of [1, 0, 0, 0, 2] is converted to [1, -3, 2].
602 """
603
604 count = 0
605
606 for value in iterable:
607 if value == 0:
608 count += 1
609 else:
610 if count != 0:
611 yield -count
612 count = 0
613 yield value
614 480
615 def add(self, value, fields=None, target_fields=None): 481 def add(self, value, fields=None, target_fields=None):
616 def modify_fn(dist, value): 482 def modify_fn(dist, value):
617 if dist == 0: 483 if dist == 0:
618 dist = distribution.Distribution(self.bucketer) 484 dist = distribution.Distribution(self.bucketer)
619 dist.add(value) 485 dist.add(value)
620 return dist 486 return dist
621 487
622 self._incr(fields, target_fields, value, modify_fn=modify_fn) 488 self._incr(fields, target_fields, value, modify_fn=modify_fn)
623 489
(...skipping 15 matching lines...) Expand all
639 505
640 self._set(fields, target_fields, value) 506 self._set(fields, target_fields, value)
641 507
642 def is_cumulative(self): 508 def is_cumulative(self):
643 return self._is_cumulative 509 return self._is_cumulative
644 510
645 511
646 class CumulativeDistributionMetric(_DistributionMetricBase): 512 class CumulativeDistributionMetric(_DistributionMetricBase):
647 """A DistributionMetric with is_cumulative set to True.""" 513 """A DistributionMetric with is_cumulative set to True."""
648 514
649 def __init__(self, name, description, field_spec=None, bucketer=None, 515 def __init__(self, name, description, field_spec, bucketer=None, units=None):
650 units=None):
651 super(CumulativeDistributionMetric, self).__init__( 516 super(CumulativeDistributionMetric, self).__init__(
652 name, description, field_spec, 517 name, description, field_spec,
653 is_cumulative=True, 518 is_cumulative=True,
654 bucketer=bucketer, 519 bucketer=bucketer,
655 units=units) 520 units=units)
656 521
657 522
658 class NonCumulativeDistributionMetric(_DistributionMetricBase): 523 class NonCumulativeDistributionMetric(_DistributionMetricBase):
659 """A DistributionMetric with is_cumulative set to False.""" 524 """A DistributionMetric with is_cumulative set to False."""
660 525
661 def __init__(self, name, description, field_spec=None, bucketer=None, 526 def __init__(self, name, description, field_spec, bucketer=None, units=None):
662 units=None):
663 super(NonCumulativeDistributionMetric, self).__init__( 527 super(NonCumulativeDistributionMetric, self).__init__(
664 name, description, field_spec, 528 name, description, field_spec,
665 is_cumulative=False, 529 is_cumulative=False,
666 bucketer=bucketer, 530 bucketer=bucketer,
667 units=units) 531 units=units)
668 532
669 533
670 class MetaMetricsDataUnits(type): 534 class MetricsDataUnits(object):
671 """Metaclass to populate the enum values of metrics_pb2.MetricsData.Units.""" 535 """An container for units of measurement for Metrics data."""
672 def __new__(mcs, name, bases, attrs):
673 attrs.update(metrics_pb2.MetricsData.Units.items())
674 return super(MetaMetricsDataUnits, mcs).__new__(mcs, name, bases, attrs)
675 536
676 537 UNKNOWN_UNITS = '{unknown}'
677 class MetricsDataUnits(object): 538 SECONDS = 's'
678 """An enumeration class for units of measurement for Metrics data. 539 MILLISECONDS = 'ms'
679 See infra_libs/ts_mon/protos/metrics.proto for a full list of supported units. 540 MICROSECONDS = 'us'
680 """ 541 NANOSECONDS = 'ns'
681 __metaclass__ = MetaMetricsDataUnits 542 BITS = 'B'
682 543 BYTES = 'By'
683 _UNITS_TO_STRING = { 544 KILOBYTES = 'kBy'
684 MetricsDataUnits.UNKNOWN_UNITS: '{unknown}', 545 MEGABYTES = 'MBy'
685 MetricsDataUnits.SECONDS: 's', 546 GIGABYTES = 'GBy'
686 MetricsDataUnits.MILLISECONDS: 'ms', 547 KIBIBYTES = 'kiBy'
687 MetricsDataUnits.MICROSECONDS: 'us', 548 MEBIBYTES = 'MiBy'
688 MetricsDataUnits.NANOSECONDS: 'ns', 549 GIBIBYTES = 'GiBy'
689 MetricsDataUnits.BITS: 'B', 550 AMPS = 'A'
690 MetricsDataUnits.BYTES: 'By', 551 MILLIAMPS = 'mA'
691 MetricsDataUnits.KILOBYTES: 'kBy', 552 DEGREES_CELSIUS = 'Cel'
692 MetricsDataUnits.MEGABYTES: 'MBy',
693 MetricsDataUnits.GIGABYTES: 'GBy',
694 MetricsDataUnits.KIBIBYTES: 'kiBy',
695 MetricsDataUnits.MEBIBYTES: 'MiBy',
696 MetricsDataUnits.GIBIBYTES: 'GiBy',
697 MetricsDataUnits.AMPS: 'A',
698 MetricsDataUnits.MILLIAMPS : 'mA',
699 MetricsDataUnits.DEGREES_CELSIUS: 'Cel'
700 }
OLDNEW
« no previous file with comments | « client/third_party/infra_libs/ts_mon/common/metric_store.py ('k') | client/third_party/infra_libs/ts_mon/common/monitors.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698