Allow no more than 100 instances to be created on each resizing run

appengine/gce-backend/instance_group_managers.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Utilities for operating on instance group managers."""
 
 import logging
 
 from google.appengine.ext import ndb
 
@@ skipping 116 matching lines @@
         'InstanceTemplateRevision project unspecified: %s', key.parent())
     return
 
   if not parent.url:
     logging.warning(
         'InstanceTemplateRevision URL unspecified: %s', key.parent())
     return
 
   api = gce.Project(parent.project)
   try:
+    # Create the instance group manager with 0 instances. The resize cron job
+    # will adjust this later.
     result = api.create_instance_group_manager(
         get_name(entity),
         parent.url,
-        entity.minimum_size,
+        0,
         entity.key.id(),
         base_name=get_base_name(entity),
     )
   except net.Error as e:
     if e.status_code == 409:
       # If the instance template already exists, just record the URL.
       result = api.get_instance_group_manager(get_name(entity), entity.key.id())
       update_url(entity.key, result['selfLink'])
       return
     else:
@@ skipping 123 matching lines @@
         logging.warning(
           'Failed to enqueue task for InstanceGroupManager: %s', key)
 
 
 def resize(key):
   """Resizes the given instance group manager.
 
   Args:
     key: ndb.Key for a models.InstanceGroupManager entity.
   """
+  # To avoid a massive resize, impose a limit on how much larger we can
+  # resize the instance group. Repeated calls will eventually allow the
+  # instance group to reach its target size. Cron timing together with
+  # this limit controls the rate at which instances are created.
+  RESIZE_LIMIT = 100
+
   entity = key.get()
   if not entity:
     logging.warning('InstanceGroupManager does not exist: %s', key)
     return
 
   if not entity.url:
     logging.warning('InstanceGroupManager URL unspecified: %s', key)
 
   parent = key.parent().get()
   if not parent:
     logging.warning('InstanceTemplateRevision does not exist: %s', key)
     return
 
   if not parent.project:
     logging.warning('InstanceTemplateRevision project unspecified: %s', key)
     return
 
-  # For now, just ensure a minimum size.
-  if entity.current_size >= entity.minimum_size:
+  api = gce.Project(parent.project)
+  response = api.get_instance_group_manager(get_name(entity), key.id())
+
+  # Try to reach the target size, but avoid increasing the number of
+  # instances by more than the resize limit. For now, the target size
+  # is just the minimum size.
+  target_size = min(entity.minimum_size, response['targetSize'] + RESIZE_LIMIT)
+  if response['targetSize'] >= target_size:

[Vadim Sh., 2016/06/29 23:17:48]

how is it going to scale down then if we change minimum_size?

[smut, 2016/06/29 23:24:38]

We never scale down, we just wait for excess VMs to drain away.

So for example if target size is 10 and then we change target size to 5 then we
will stay at 10 until 5 VM leases expire and we delete them. Deleting a VM from
an instance group decreases size, so we will eventually come down to 5.

     return
 
-  api = gce.Project(parent.project)
-  api.resize_managed_instance_group(
-      get_name(entity), key.id(), entity.minimum_size)
+  api.resize_managed_instance_group(response['name'], key.id(), target_size)
 
 
 def schedule_resize():
   """Enqueues tasks to resize instance group managers."""
   for instance_template in models.InstanceTemplate.query():
     if instance_template.active:
       instance_template_revision = instance_template.active.get()
       if instance_template_revision:
         for instance_group_manager_key in instance_template_revision.active:
           if not utils.enqueue_task(
               '/internal/queues/resize-instance-group',
               'resize-instance-group',
               params={
                   'key': instance_group_manager_key.urlsafe(),
               },
           ):
             logging.warning(
                 'Failed to enqueue task for InstanceGroupManager: %s',
                 instance_group_manager_key,
             )