Roll infra_libs and gae_ts_mon in luci-py, and add field_specs to all metrics

appengine/swarming/ts_mon_metrics.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Timeseries metrics."""
 
 from collections import defaultdict
 import datetime
 import itertools
 import json
@@ skipping 19 matching lines @@
 # Override default target fields for app-global metrics.
 TARGET_FIELDS = {
     'job_name':  '',  # module name
     'hostname': '',  # version
     'task_num':  0,  # instance ID
 }
 
 # A custom bucketer with 12% resolution in the range of 1..10**5.
 # Used for job cycle times.
 _bucketer = gae_ts_mon.GeometricBucketer(growth_factor=10**0.05,
-                                     num_finite_buckets=100)
+                                         num_finite_buckets=100)
 
 # Regular (instance-local) metrics: jobs/completed and jobs/durations.
 # Both have the following metric fields:
 # - project_id: e.g. 'chromium'
 # - subproject_id: e.g. 'blink'. Set to empty string if not used.
 # - spec_name: name of a job specification, e.g. '<master>:<builder>:<test>'
 #     for buildbot jobs.
 # - result: one of 'success', 'failure', or 'infra-failure'.
 jobs_completed = gae_ts_mon.CounterMetric(
     'jobs/completed',
-    description='Number of completed jobs.')
+    'Number of completed jobs.', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+        gae_ts_mon.StringField('result'),
+    ])
 
 
 jobs_durations = gae_ts_mon.CumulativeDistributionMetric(
-    'jobs/durations', bucketer=_bucketer,
-    description='Cycle times of completed jobs, in seconds.')
+    'jobs/durations',
+    'Cycle times of completed jobs, in seconds.', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+        gae_ts_mon.StringField('result'),
+    ],
+    bucketer=_bucketer)
 
 
 # Similar to jobs/completed and jobs/duration, but with a dedup field.
 # - project_id: e.g. 'chromium'
 # - subproject_id: e.g. 'blink'. Set to empty string if not used.
 # - spec_name: name of a job specification, e.g. '<master>:<builder>:<test>'
 #     for buildbot jobs.
 # - deduped: boolean describing whether the job was deduped or not.
 jobs_requested = gae_ts_mon.CounterMetric(
     'jobs/requested',
-    description='Number of requested jobs over time.')
+    'Number of requested jobs over time.', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+        gae_ts_mon.BooleanField('deduped'),
+    ])
 
 
 # Swarming-specific metric. Metric fields:
 # - project_id: e.g. 'chromium'
 # - subproject_id: e.g. 'blink'. Set to empty string if not used.
 # - spec_name: name of a job specification, e.g. '<master>:<builder>:<test>'
 #     for buildbot jobs.
 tasks_expired = gae_ts_mon.CounterMetric(
     'swarming/tasks/expired',
-    description='Number of expired tasks')
+    'Number of expired tasks', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+    ])
 
 # Global metric. Metric fields:
 # - project_id: e.g. 'chromium'
 # - subproject_id: e.g. 'blink'. Set to empty string if not used.
 # - spec_name: name of a job specification, e.g. '<master>:<builder>:<test>'
 #     for buildbot jobs.
 # Override target field:
 # - hostname: 'autogen:<executor_id>': name of the bot that executed a job,
 #     or an empty string. e.g. 'autogen:swarm42-m4'.
 # Value should be 'pending' or 'running'. Completed / canceled jobs should not
 # send this metric.
 jobs_running = gae_ts_mon.BooleanMetric(
     'jobs/running',
-    description='Presence metric for a running job.')
+    'Presence metric for a running job.', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+    ])
 
 # Global metric. Metric fields:
 # - project_id: e.g. 'chromium'
 # - subproject_id: e.g. 'blink'. Set to empty string if not used.
 # - spec_name: name of a job specification, e.g. '<master>:<builder>:<test>'
 #     for buildbot jobs.
 # - status: 'pending' or 'running'.
 jobs_active = gae_ts_mon.GaugeMetric(
     'jobs/active',
-    description='Number of running, pending or otherwise active jobs.')
+    'Number of running, pending or otherwise active jobs.', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+        gae_ts_mon.StringField('status'),
+    ])
 
 
 # Global metric. Target field: hostname = 'autogen:<executor_id>' (bot id).
 executors_pool = gae_ts_mon.StringMetric(
     'executors/pool',
-    description='Pool name for a given job executor.')
+    'Pool name for a given job executor.',
+    None)
 
 
 # Global metric. Target fields:
 # - hostname = 'autogen:<executor_id>' (bot id).
 # Status value must be 'ready', 'running', or anything else, possibly
 # swarming-specific, when it cannot run a job. E.g. 'quarantined' or
 # 'dead'.
 executors_status = gae_ts_mon.StringMetric(
     'executors/status',
-    description=('Status of a job executor.'))
+    'Status of a job executor.',
+    None)
 
 
 # Global metric. Target fields:
 # - hostname = 'autogen:<executor_id>' (bot id).
 # Status value must be 'ready', 'running', or anything else, possibly
 # swarming-specific, when it cannot run a job. E.g. 'quarantined' or
 # 'dead'.
 # Note that 'running' will report data as long as the job is running,
 # so it is best to restrict data to status == 'pending.'
 jobs_pending_durations = gae_ts_mon.NonCumulativeDistributionMetric(
-    'jobs/pending_durations', bucketer=_bucketer,
-    description='Pending times of active jobs, in seconds.')
+    'jobs/pending_durations',
+    'Pending times of active jobs, in seconds.', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+        gae_ts_mon.StringField('status'),
+    ],
+    bucketer=_bucketer)
 
 
 # Global metric. Target fields:
 # - hostname = 'autogen:<executor_id>' (bot id).
 # Status value must be 'ready', 'running', or anything else, possibly
 # swarming-specific, when it cannot run a job. E.g. 'quarantined' or
 # 'dead'.
 # Note that 'running' will report data as long as the job is running,
 # so it is best to restrict data to status == 'pending.'
 jobs_max_pending_duration = gae_ts_mon.FloatMetric(
     'jobs/max_pending_duration',
-    description='Maximum pending seconds of pending jobs.')
+    'Maximum pending seconds of pending jobs.', [
+        gae_ts_mon.StringField('spec_name'),
+        gae_ts_mon.StringField('project_id'),
+        gae_ts_mon.StringField('subproject_id'),
+        gae_ts_mon.StringField('status'),
+    ])
 
 
 # Global metric. Target fields:
 # - machine_type = server.lease_management.MachineType.key.id().
 # - enabled = server.lease_management.MachineType.enabled.
 machine_types_target_size = gae_ts_mon.GaugeMetric(
     'swarming/machine_types/target_size',
-    description='Target size of each MachineType leased from Machine Provider.')
+    'Target size of each MachineType leased from Machine Provider.', [
+        gae_ts_mon.BooleanField('enabled'),
+        gae_ts_mon.StringField('machine_type'),
+    ])
 
 
 def pool_from_dimensions(dimensions):
   """Return a canonical string of flattened dimensions."""
   iterables = (map(lambda x: '%s:%s' % (key, x), values)
                for key, values in dimensions.iteritems()
                if key not in IGNORED_DIMENSIONS)
   return '|'.join(sorted(itertools.chain(*iterables)))
 
 
@@ skipping 215 matching lines @@
   gae_ts_mon.register_global_metrics([
       executors_pool,
       executors_status,
       jobs_active,
       jobs_max_pending_duration,
       jobs_pending_durations,
       jobs_running,
       machine_types_target_size,
   ])
   gae_ts_mon.register_global_metrics_callback('callback', _set_global_metrics)