Count instances managed by each instance template

appengine/gce-backend/metrics.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Metrics to track with ts_mon and event_mon."""
 
 import logging
 
 import gae_event_mon
 import gae_ts_mon
 
+import instance_group_managers
 import instances
 
 
 # Overrides to create app-global metrics.
 GLOBAL_TARGET_FIELDS = {
   # Name of the module reporting the metric.
   'job_name': '',
   # Version of the app reporting the metric.
   'hostname': '',
   # ID of the instance reporting the metric.
   'task_num': 0,
 }
 
 
 GLOBAL_METRICS = {
     'instances': gae_ts_mon.GaugeMetric(
         'machine_provider/gce_backend/instances',
         description='Current count of the number of instances.',
     ),
+    'instances/v2': gae_ts_mon.GaugeMetric(
+        'machine_provider/gce_backend/instances/v2',

[Sergey Berezin (google), 2016/11/23 02:28:25]

I'd name it in a more self-descriptive way, e.g. .../instances_by_template.
Or, even better, just add a new field to the existing metric and count
orphaned/non-orphaned x templates.

[smut, 2016/11/23 22:50:57]

I tried modifying the existing metric and the metric stopped showing up at all.
I thought you once said that field schema shouldn't be changed?

At any rate, the orphaned/unorphaned distinction is somewhat meaningless. An
implementation detail causes tons of orphaned instances to appear in the minutes
following instance deletion. Originally I was counting orphans to find errors,
but now it's standard operating procedure to detect orphans.

[Sergey Berezin (google), 2016/11/23 23:42:44]

No, the field schema should be consistent globally - at least that was a
requirement previously, and apparently now missing fields don't (always) cause
the metric to fail. I'm pretty sure it's OK to change the fields if you redeploy
all the instances that send the metric. Some data may be lost for the transition
period though.
Do you care about the current history of this metric? If not, I'd say let's
update the metric with the new field, drop the 'orphaned', and once deployed,
we'll delete the metric on backend and it will be recreated in its new form. The
reason is, it's a very good name that otherwise would be waisted and be
confusing to everyone. Best to reuse it if we can. WDYT?

[smut, 2016/11/23 23:51:09]

No, I don't care about the history. I would prefer this be the only "instances"
metric. Uploaded new patch, and deployed on staging and prod. Should be safe to
delete the old metric's history now and let the new one populate.

[Sergey Berezin (google), 2016/11/24 03:51:47]

Great, I deleted the metric - once the update propagates, we'll hopefully see
just one metric field. Thanks!

+        description='Current count of the number of instances.',
+    ),
 }
 
 
 config_valid = gae_ts_mon.BooleanMetric(
     'machine_provider/gce_backend/config/valid',
     description='Whether or not the current config is valid.',
 )
 
 
 def compute_global_metrics():
   orphaned, total = instances.count_instances()
   GLOBAL_METRICS['instances'].set(
       orphaned,
       fields={
           'orphaned': True,
       },
       target_fields=GLOBAL_TARGET_FIELDS,
   )
   GLOBAL_METRICS['instances'].set(
       total - orphaned,
       fields={
           'orphaned': False,
       },
       target_fields=GLOBAL_TARGET_FIELDS,
   )
+  for name, count in instance_group_managers.count_instances().iteritems():
+    logging.info('%s: %s', name, count)
+    GLOBAL_METRICS['instances/v2'].set(
+        count,
+        fields={
+            'instance_template': name,
+        },
+        target_fields=GLOBAL_TARGET_FIELDS,
+    )
 
 
 def initialize():
   gae_ts_mon.register_global_metrics(GLOBAL_METRICS.values())
   gae_ts_mon.register_global_metrics_callback(
       'callback', compute_global_metrics)
 
 
 def send_machine_event(state, hostname):
   """Sends an event_mon event about a GCE instance.
 
   Args:
     state: gae_event_mon.ChromeInfraEvent.GCEBackendMachineState.
     hostname: Name of the GCE instance this event is for.
   """
   state = gae_event_mon.MachineProviderEvent.GCEBackendMachineState.Value(state)
   event = gae_event_mon.Event('POINT')
   event.proto.event_source.host_name = hostname
   event.proto.machine_provider_event.gce_backend_state = state
   logging.info('Sending event: %s', event.proto)
   event.send()