| OLD | NEW |
| 1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
| 2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
| 3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
| 4 | 4 |
| 5 """Utilities for cleaning up GCE Backend.""" | 5 """Utilities for cleaning up GCE Backend.""" |
| 6 | 6 |
| 7 import datetime | 7 import datetime |
| 8 import logging | 8 import logging |
| 9 | 9 |
| 10 from google.appengine.ext import ndb | 10 from google.appengine.ext import ndb |
| (...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 221 | 221 |
| 222 if not exists(instance.url): | 222 if not exists(instance.url): |
| 223 # When the instance isn't found, assume it's deleted. | 223 # When the instance isn't found, assume it's deleted. |
| 224 set_instance_deleted(key, False) | 224 set_instance_deleted(key, False) |
| 225 | 225 |
| 226 | 226 |
| 227 def schedule_deleted_instance_check(): | 227 def schedule_deleted_instance_check(): |
| 228 """Enqueues tasks to check for deleted instances.""" | 228 """Enqueues tasks to check for deleted instances.""" |
| 229 for instance in models.Instance.query(): | 229 for instance in models.Instance.query(): |
| 230 if instance.pending_deletion and not instance.deleted: | 230 if instance.pending_deletion and not instance.deleted: |
| 231 if not utils.enqueue_task( | 231 utilities.enqueue_task('check-deleted-instance', instance.key) |
| 232 '/internal/queues/check-deleted-instance', | |
| 233 'check-deleted-instance', | |
| 234 params={ | |
| 235 'key': instance.key.urlsafe(), | |
| 236 }, | |
| 237 ): | |
| 238 logging.warning('Failed to enqueue task for Instance: %s', instance.key) | |
| 239 | 232 |
| 240 | 233 |
| 241 @ndb.transactional | 234 @ndb.transactional |
| 242 def cleanup_deleted_instance(key): | 235 def cleanup_deleted_instance(key): |
| 243 """Deletes the given Instance. | 236 """Deletes the given Instance. |
| 244 | 237 |
| 245 Args: | 238 Args: |
| 246 key: ndb.Key for a models.Instance entity. | 239 key: ndb.Key for a models.Instance entity. |
| 247 """ | 240 """ |
| 248 instance = key.get() | 241 instance = key.get() |
| (...skipping 17 matching lines...) Expand all Loading... |
| 266 # which gets deleted before it finishes, and at the end of the queue it may | 259 # which gets deleted before it finishes, and at the end of the queue it may |
| 267 # incorrectly create an entity for that deleted instance. Since task queues | 260 # incorrectly create an entity for that deleted instance. Since task queues |
| 268 # can take at most 10 minutes, we can avoid the race condition by deleting | 261 # can take at most 10 minutes, we can avoid the race condition by deleting |
| 269 # only those entities referring to instances which were detected as having | 262 # only those entities referring to instances which were detected as having |
| 270 # been deleted >10 minutes ago. Here we use 20 minutes for safety. | 263 # been deleted >10 minutes ago. Here we use 20 minutes for safety. |
| 271 THRESHOLD = 60 * 20 | 264 THRESHOLD = 60 * 20 |
| 272 now = utils.utcnow() | 265 now = utils.utcnow() |
| 273 | 266 |
| 274 for instance in models.Instance.query(): | 267 for instance in models.Instance.query(): |
| 275 if instance.deleted and (now - instance.last_updated).seconds > THRESHOLD: | 268 if instance.deleted and (now - instance.last_updated).seconds > THRESHOLD: |
| 276 if not utils.enqueue_task( | 269 utilities.enqueue_task('cleanup-deleted-instance', instance.key) |
| 277 '/internal/queues/cleanup-deleted-instance', | |
| 278 'cleanup-deleted-instance', | |
| 279 params={ | |
| 280 'key': instance.key.urlsafe(), | |
| 281 }, | |
| 282 ): | |
| 283 logging.warning('Failed to enqueue task for Instance: %s', instance.key) | |
| 284 | 270 |
| 285 | 271 |
| 286 def cleanup_drained_instance(key): | 272 def cleanup_drained_instance(key): |
| 287 """Deletes the given drained Instance. | 273 """Deletes the given drained Instance. |
| 288 | 274 |
| 289 Args: | 275 Args: |
| 290 key: ndb.Key for a models.Instance entity. | 276 key: ndb.Key for a models.Instance entity. |
| 291 """ | 277 """ |
| 292 instance = key.get() | 278 instance = key.get() |
| 293 if not instance: | 279 if not instance: |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 333 | 319 |
| 334 def schedule_drained_instance_cleanup(): | 320 def schedule_drained_instance_cleanup(): |
| 335 """Enqueues tasks to clean up drained instances.""" | 321 """Enqueues tasks to clean up drained instances.""" |
| 336 for instance_group_manager_key in ( | 322 for instance_group_manager_key in ( |
| 337 instance_group_managers.get_drained_instance_group_managers()): | 323 instance_group_managers.get_drained_instance_group_managers()): |
| 338 instance_group_manager = instance_group_manager_key.get() | 324 instance_group_manager = instance_group_manager_key.get() |
| 339 if instance_group_manager: | 325 if instance_group_manager: |
| 340 for instance_key in instance_group_manager.instances: | 326 for instance_key in instance_group_manager.instances: |
| 341 instance = instance_key.get() | 327 instance = instance_key.get() |
| 342 if instance and not instance.cataloged: | 328 if instance and not instance.cataloged: |
| 343 if not utils.enqueue_task( | 329 utilities.enqueue_task('cleanup-drained-instance', instance.key) |
| 344 '/internal/queues/cleanup-drained-instance', | |
| 345 'cleanup-drained-instance', | |
| 346 params={ | |
| 347 'key': instance.key.urlsafe(), | |
| 348 }, | |
| 349 ): | |
| 350 logging.warning( | |
| 351 'Failed to enqueue task for Instance: %s', instance.key) | |
| OLD | NEW |