OLD | NEW |
| (Empty) |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 """Classes representing individual metrics that can be sent.""" | |
6 | |
7 import copy | |
8 | |
9 from infra_libs.ts_mon.protos import metrics_pb2 | |
10 | |
11 from infra_libs.ts_mon.common import distribution | |
12 from infra_libs.ts_mon.common import errors | |
13 from infra_libs.ts_mon.common import interface | |
14 | |
15 | |
16 MICROSECONDS_PER_SECOND = 1000000 | |
17 | |
18 | |
19 class Metric(object): | |
20 """Abstract base class for a metric. | |
21 | |
22 A Metric is an attribute that may be monitored across many targets. Examples | |
23 include disk usage or the number of requests a server has received. A single | |
24 process may keep track of many metrics. | |
25 | |
26 Note that Metric objects may be initialized at any time (for example, at the | |
27 top of a library), but cannot be sent until the underlying Monitor object | |
28 has been set up (usually by the top-level process parsing the command line). | |
29 | |
30 A Metric can actually store multiple values that are identified by a set of | |
31 fields (which are themselves key-value pairs). Fields can be passed to the | |
32 set() or increment() methods to modify a particular value, or passed to the | |
33 constructor in which case they will be used as the defaults for this Metric. | |
34 | |
35 The unit of measurement for Metric data can be specified with MetricsDataUnits | |
36 when a Metric object is created: | |
37 e.g., MetricsDataUnits.SECONDS, MetricsDataUnits.BYTES, and etc.., | |
38 A full list of supported units can be found in the following protobuf file | |
39 : infra_libs/ts_mon/protos/metrics.proto | |
40 | |
41 Do not directly instantiate an object of this class. | |
42 Use the concrete child classes instead: | |
43 * StringMetric for metrics with string value | |
44 * BooleanMetric for metrics with boolean values | |
45 * CounterMetric for metrics with monotonically increasing integer values | |
46 * GaugeMetric for metrics with arbitrarily varying integer values | |
47 * CumulativeMetric for metrics with monotonically increasing float values | |
48 * FloatMetric for metrics with arbitrarily varying float values | |
49 | |
50 See http://go/inframon-doc for help designing and using your metrics. | |
51 """ | |
52 | |
53 def __init__(self, name, fields=None, description=None, units=None): | |
54 """Create an instance of a Metric. | |
55 | |
56 Args: | |
57 name (str): the file-like name of this metric | |
58 fields (dict): a set of key-value pairs to be set as default metric fields | |
59 description (string): help string for the metric. Should be enough to | |
60 know what the metric is about. | |
61 units (int): the unit used to measure data for given | |
62 metric. Please use the attributes of MetricDataUnit to find | |
63 valid integer values for this argument. | |
64 """ | |
65 self._name = name.lstrip('/') | |
66 self._start_time = None | |
67 fields = fields or {} | |
68 if len(fields) > 7: | |
69 raise errors.MonitoringTooManyFieldsError(self._name, fields) | |
70 self._fields = fields | |
71 self._normalized_fields = self._normalize_fields(self._fields) | |
72 self._description = description | |
73 self._units = units | |
74 | |
75 interface.register(self) | |
76 | |
77 @property | |
78 def name(self): | |
79 return self._name | |
80 | |
81 @property | |
82 def start_time(self): | |
83 return self._start_time | |
84 | |
85 def is_cumulative(self): | |
86 raise NotImplementedError() | |
87 | |
88 def __eq__(self, other): | |
89 return (self.name == other.name and | |
90 self._fields == other._fields and | |
91 type(self) == type(other)) | |
92 | |
93 def unregister(self): | |
94 interface.unregister(self) | |
95 | |
96 def serialize_to(self, collection_pb, start_time, fields, value, target): | |
97 """Generate metrics_pb2.MetricsData messages for this metric. | |
98 | |
99 Args: | |
100 collection_pb (metrics_pb2.MetricsCollection): protocol buffer into which | |
101 to add the current metric values. | |
102 start_time (int): timestamp in microseconds since UNIX epoch. | |
103 target (Target): a Target to use. | |
104 """ | |
105 | |
106 metric_pb = collection_pb.data.add() | |
107 metric_pb.metric_name_prefix = interface.state.metric_name_prefix | |
108 metric_pb.name = self._name | |
109 if self._description is not None: | |
110 metric_pb.description = self._description | |
111 if self._units is not None: | |
112 metric_pb.units = self._units | |
113 | |
114 self._populate_value(metric_pb, value, start_time) | |
115 self._populate_fields(metric_pb, fields) | |
116 | |
117 target._populate_target_pb(metric_pb) | |
118 | |
119 def _populate_fields(self, metric, fields): | |
120 """Fill in the fields attribute of a metric protocol buffer. | |
121 | |
122 Args: | |
123 metric (metrics_pb2.MetricsData): a metrics protobuf to populate | |
124 fields (list of (key, value) tuples): normalized metric fields | |
125 | |
126 Raises: | |
127 MonitoringInvalidFieldTypeError: if a field has a value of unknown type | |
128 """ | |
129 for key, value in fields: | |
130 field = metric.fields.add() | |
131 field.name = key | |
132 if isinstance(value, basestring): | |
133 field.type = metrics_pb2.MetricsField.STRING | |
134 field.string_value = value | |
135 elif isinstance(value, bool): | |
136 field.type = metrics_pb2.MetricsField.BOOL | |
137 field.bool_value = value | |
138 elif isinstance(value, int): | |
139 field.type = metrics_pb2.MetricsField.INT | |
140 field.int_value = value | |
141 else: | |
142 raise errors.MonitoringInvalidFieldTypeError(self._name, key, value) | |
143 | |
144 def _normalize_fields(self, fields): | |
145 """Merges the fields with the default fields and returns something hashable. | |
146 | |
147 Args: | |
148 fields (dict): A dict of fields passed by the user, or None. | |
149 | |
150 Returns: | |
151 A tuple of (key, value) tuples, ordered by key. This whole tuple is used | |
152 as the key in the self._values dict to identify the cell for a value. | |
153 | |
154 Raises: | |
155 MonitoringTooManyFieldsError: if there are more than seven metric fields | |
156 """ | |
157 if fields is None: | |
158 return self._normalized_fields | |
159 | |
160 all_fields = copy.copy(self._fields) | |
161 all_fields.update(fields) | |
162 | |
163 if len(all_fields) > 7: | |
164 raise errors.MonitoringTooManyFieldsError(self._name, all_fields) | |
165 | |
166 return tuple(sorted(all_fields.iteritems())) | |
167 | |
168 def _populate_value(self, metric, value, start_time): | |
169 """Fill in the the data values of a metric protocol buffer. | |
170 | |
171 Args: | |
172 metric (metrics_pb2.MetricsData): a metrics protobuf to populate | |
173 value (see concrete class): the value of the metric to be set | |
174 start_time (int): timestamp in microseconds since UNIX epoch. | |
175 """ | |
176 raise NotImplementedError() | |
177 | |
178 def set(self, value, fields=None, target_fields=None): | |
179 """Set a new value for this metric. Results in sending a new value. | |
180 | |
181 The subclass should do appropriate type checking on value and then call | |
182 self._set_and_send_value. | |
183 | |
184 Args: | |
185 value (see concrete class): the value of the metric to be set | |
186 fields (dict): additional metric fields to complement those on self | |
187 target_fields (dict): overwrite some of the default target fields | |
188 """ | |
189 raise NotImplementedError() | |
190 | |
191 def get(self, fields=None, target_fields=None): | |
192 """Returns the current value for this metric. | |
193 | |
194 Subclasses should never use this to get a value, modify it and set it again. | |
195 Instead use _incr with a modify_fn. | |
196 """ | |
197 return interface.state.store.get( | |
198 self.name, self._normalize_fields(fields), target_fields) | |
199 | |
200 def get_all(self): | |
201 return interface.state.store.iter_field_values(self.name) | |
202 | |
203 def reset(self): | |
204 """Clears the values of this metric. Useful in unit tests. | |
205 | |
206 It might be easier to call ts_mon.reset_for_unittest() in your setUp() | |
207 method instead of resetting every individual metric. | |
208 """ | |
209 | |
210 interface.state.store.reset_for_unittest(self.name) | |
211 | |
212 def _set(self, fields, target_fields, value, enforce_ge=False): | |
213 interface.state.store.set(self.name, self._normalize_fields(fields), | |
214 target_fields, value, enforce_ge=enforce_ge) | |
215 | |
216 def _incr(self, fields, target_fields, delta, modify_fn=None): | |
217 interface.state.store.incr(self.name, self._normalize_fields(fields), | |
218 target_fields, delta, modify_fn=modify_fn) | |
219 | |
220 | |
221 class StringMetric(Metric): | |
222 """A metric whose value type is a string.""" | |
223 | |
224 def _populate_value(self, metric, value, start_time): | |
225 metric.string_value = value | |
226 | |
227 def set(self, value, fields=None, target_fields=None): | |
228 if not isinstance(value, basestring): | |
229 raise errors.MonitoringInvalidValueTypeError(self._name, value) | |
230 self._set(fields, target_fields, value) | |
231 | |
232 def is_cumulative(self): | |
233 return False | |
234 | |
235 | |
236 class BooleanMetric(Metric): | |
237 """A metric whose value type is a boolean.""" | |
238 | |
239 def _populate_value(self, metric, value, start_time): | |
240 metric.boolean_value = value | |
241 | |
242 def set(self, value, fields=None, target_fields=None): | |
243 if not isinstance(value, bool): | |
244 raise errors.MonitoringInvalidValueTypeError(self._name, value) | |
245 self._set(fields, target_fields, value) | |
246 | |
247 def is_cumulative(self): | |
248 return False | |
249 | |
250 | |
251 class NumericMetric(Metric): # pylint: disable=abstract-method | |
252 """Abstract base class for numeric (int or float) metrics.""" | |
253 # TODO(agable): Figure out if there's a way to send units with these metrics. | |
254 | |
255 def increment(self, fields=None, target_fields=None): | |
256 self._incr(fields, target_fields, 1) | |
257 | |
258 def increment_by(self, step, fields=None, target_fields=None): | |
259 self._incr(fields, target_fields, step) | |
260 | |
261 | |
262 class CounterMetric(NumericMetric): | |
263 """A metric whose value type is a monotonically increasing integer.""" | |
264 | |
265 def __init__(self, name, fields=None, start_time=None, description=None, | |
266 units=None): | |
267 super(CounterMetric, self).__init__( | |
268 name, fields=fields, description=description, units=units) | |
269 self._start_time = start_time | |
270 | |
271 def _populate_value(self, metric, value, start_time): | |
272 metric.counter = value | |
273 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND) | |
274 | |
275 def set(self, value, fields=None, target_fields=None): | |
276 if not isinstance(value, (int, long)): | |
277 raise errors.MonitoringInvalidValueTypeError(self._name, value) | |
278 self._set(fields, target_fields, value, enforce_ge=True) | |
279 | |
280 def increment_by(self, step, fields=None, target_fields=None): | |
281 if not isinstance(step, (int, long)): | |
282 raise errors.MonitoringInvalidValueTypeError(self._name, step) | |
283 self._incr(fields, target_fields, step) | |
284 | |
285 def is_cumulative(self): | |
286 return True | |
287 | |
288 | |
289 class GaugeMetric(NumericMetric): | |
290 """A metric whose value type is an integer.""" | |
291 | |
292 def _populate_value(self, metric, value, start_time): | |
293 metric.gauge = value | |
294 | |
295 def set(self, value, fields=None, target_fields=None): | |
296 if not isinstance(value, (int, long)): | |
297 raise errors.MonitoringInvalidValueTypeError(self._name, value) | |
298 self._set(fields, target_fields, value) | |
299 | |
300 def is_cumulative(self): | |
301 return False | |
302 | |
303 | |
304 class CumulativeMetric(NumericMetric): | |
305 """A metric whose value type is a monotonically increasing float.""" | |
306 | |
307 def __init__(self, name, fields=None, start_time=None, description=None, | |
308 units=None): | |
309 super(CumulativeMetric, self).__init__( | |
310 name, fields=fields, description=description, units=units) | |
311 self._start_time = start_time | |
312 | |
313 def _populate_value(self, metric, value, start_time): | |
314 metric.cumulative_double_value = value | |
315 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND) | |
316 | |
317 def set(self, value, fields=None, target_fields=None): | |
318 if not isinstance(value, (float, int)): | |
319 raise errors.MonitoringInvalidValueTypeError(self._name, value) | |
320 self._set(fields, target_fields, float(value), enforce_ge=True) | |
321 | |
322 def is_cumulative(self): | |
323 return True | |
324 | |
325 | |
326 class FloatMetric(NumericMetric): | |
327 """A metric whose value type is a float.""" | |
328 | |
329 def _populate_value(self, metric, value, start_time): | |
330 metric.noncumulative_double_value = value | |
331 | |
332 def set(self, value, fields=None, target_fields=None): | |
333 if not isinstance(value, (float, int)): | |
334 raise errors.MonitoringInvalidValueTypeError(self._name, value) | |
335 self._set(fields, target_fields, float(value)) | |
336 | |
337 def is_cumulative(self): | |
338 return False | |
339 | |
340 | |
341 class DistributionMetric(Metric): | |
342 """A metric that holds a distribution of values. | |
343 | |
344 By default buckets are chosen from a geometric progression, each bucket being | |
345 approximately 1.59 times bigger than the last. In practice this is suitable | |
346 for many kinds of data, but you may want to provide a FixedWidthBucketer or | |
347 GeometricBucketer with different parameters.""" | |
348 | |
349 CANONICAL_SPEC_TYPES = { | |
350 2: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_2, | |
351 10**0.2: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10_P_0_2, | |
352 10: metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10, | |
353 } | |
354 | |
355 def __init__(self, name, is_cumulative=True, bucketer=None, fields=None, | |
356 start_time=None, description=None, units=None): | |
357 super(DistributionMetric, self).__init__( | |
358 name, fields=fields, description=description, units=units) | |
359 self._start_time = start_time | |
360 | |
361 if bucketer is None: | |
362 bucketer = distribution.GeometricBucketer() | |
363 | |
364 self._is_cumulative = is_cumulative | |
365 self.bucketer = bucketer | |
366 | |
367 def _populate_value(self, metric, value, start_time): | |
368 pb = metric.distribution | |
369 | |
370 pb.is_cumulative = self._is_cumulative | |
371 if self._is_cumulative: | |
372 metric.start_timestamp_us = int(start_time * MICROSECONDS_PER_SECOND) | |
373 | |
374 # Copy the bucketer params. | |
375 if (value.bucketer.width == 0 and | |
376 value.bucketer.growth_factor in self.CANONICAL_SPEC_TYPES): | |
377 pb.spec_type = self.CANONICAL_SPEC_TYPES[value.bucketer.growth_factor] | |
378 else: | |
379 pb.spec_type = metrics_pb2.PrecomputedDistribution.CUSTOM_PARAMETERIZED | |
380 pb.width = value.bucketer.width | |
381 pb.growth_factor = value.bucketer.growth_factor | |
382 pb.num_buckets = value.bucketer.num_finite_buckets | |
383 | |
384 # Copy the distribution bucket values. Only include the finite buckets, not | |
385 # the overflow buckets on each end. | |
386 pb.bucket.extend(self._running_zero_generator( | |
387 value.buckets.get(i, 0) for i in | |
388 xrange(1, value.bucketer.total_buckets - 1))) | |
389 | |
390 # Add the overflow buckets if present. | |
391 if value.bucketer.underflow_bucket in value.buckets: | |
392 pb.underflow = value.buckets[value.bucketer.underflow_bucket] | |
393 if value.bucketer.overflow_bucket in value.buckets: | |
394 pb.overflow = value.buckets[value.bucketer.overflow_bucket] | |
395 | |
396 if value.count != 0: | |
397 pb.mean = float(value.sum) / value.count | |
398 | |
399 @staticmethod | |
400 def _running_zero_generator(iterable): | |
401 """Compresses sequences of zeroes in the iterable into negative zero counts. | |
402 | |
403 For example an input of [1, 0, 0, 0, 2] is converted to [1, -3, 2]. | |
404 """ | |
405 | |
406 count = 0 | |
407 | |
408 for value in iterable: | |
409 if value == 0: | |
410 count += 1 | |
411 else: | |
412 if count != 0: | |
413 yield -count | |
414 count = 0 | |
415 yield value | |
416 | |
417 def add(self, value, fields=None, target_fields=None): | |
418 def modify_fn(dist, value): | |
419 if dist == 0: | |
420 dist = distribution.Distribution(self.bucketer) | |
421 dist.add(value) | |
422 return dist | |
423 | |
424 self._incr(fields, target_fields, value, modify_fn=modify_fn) | |
425 | |
426 def set(self, value, fields=None, target_fields=None): | |
427 """Replaces the distribution with the given fields with another one. | |
428 | |
429 This only makes sense on non-cumulative DistributionMetrics. | |
430 | |
431 Args: | |
432 value: A infra_libs.ts_mon.Distribution. | |
433 """ | |
434 | |
435 if self._is_cumulative: | |
436 raise TypeError( | |
437 'Cannot set() a cumulative DistributionMetric (use add() instead)') | |
438 | |
439 if not isinstance(value, distribution.Distribution): | |
440 raise errors.MonitoringInvalidValueTypeError(self._name, value) | |
441 | |
442 self._set(fields, target_fields, value) | |
443 | |
444 def is_cumulative(self): | |
445 raise NotImplementedError() # Keep this class abstract. | |
446 | |
447 | |
448 class CumulativeDistributionMetric(DistributionMetric): | |
449 """A DistributionMetric with is_cumulative set to True.""" | |
450 | |
451 def __init__(self, name, bucketer=None, fields=None, | |
452 description=None, units=None): | |
453 super(CumulativeDistributionMetric, self).__init__( | |
454 name, | |
455 is_cumulative=True, | |
456 bucketer=bucketer, | |
457 fields=fields, | |
458 description=description, | |
459 units=units) | |
460 | |
461 def is_cumulative(self): | |
462 return True | |
463 | |
464 | |
465 class NonCumulativeDistributionMetric(DistributionMetric): | |
466 """A DistributionMetric with is_cumulative set to False.""" | |
467 | |
468 def __init__(self, name, bucketer=None, fields=None, | |
469 description=None, units=None): | |
470 super(NonCumulativeDistributionMetric, self).__init__( | |
471 name, | |
472 is_cumulative=False, | |
473 bucketer=bucketer, | |
474 fields=fields, | |
475 description=description, | |
476 units=units) | |
477 | |
478 def is_cumulative(self): | |
479 return False | |
480 | |
481 | |
482 class MetaMetricsDataUnits(type): | |
483 """Metaclass to populate the enum values of metrics_pb2.MetricsData.Units.""" | |
484 def __new__(mcs, name, bases, attrs): | |
485 attrs.update(metrics_pb2.MetricsData.Units.items()) | |
486 return super(MetaMetricsDataUnits, mcs).__new__(mcs, name, bases, attrs) | |
487 | |
488 | |
489 class MetricsDataUnits(object): | |
490 """An enumeration class for units of measurement for Metrics data. | |
491 See infra_libs/ts_mon/protos/metrics.proto for a full list of supported units. | |
492 """ | |
493 __metaclass__ = MetaMetricsDataUnits | |
OLD | NEW |