Make Swarming bot keep 5%+250MB of the disk free.

appengine/swarming/swarming_bot/api/os_utilities.py

 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """OS specific utility functions.
 
 Includes code:
 - to declare the current system this code is running under.
 - to run a command on user login.
 - to restart the host.
@@ skipping 29 matching lines @@
 from third_party.oauth2client import client
 
 
 # For compatibility with older bot_config.py files.
 cached = tools.cached
 
 
 # Global configurable values. These are meant to be the default values under
 # which the bot will self-quarantine when the disk size is too small. Setting
 # either of the following value to 0 or None disables self-quarantine.
-# Self-quarantine only happens when *both* thresholds are under.
-#
+# Self-quarantine only happens when the free disk space is below the lower of
+# the two thresholds. It means that for sufficiently large disks THRESHOLD_MB is
+# the actual threshold (for them disk_size * THRESHOLD_RELATIVE > THRESHOLD_MB).
+
 # THRESHOLD_MB is the number of MiB below which the bot will quarantine itself
 # automatically.
 THRESHOLD_MB = 4*1024
+
 # THRESHOLD_RELATIVE is the minimum free space percentage under which the bot
 # will quarantine itself automatically.
+#
+# This threshold is effectively disabled for large disks (ones with
+# disk_size * THRESHOLD_RELATIVE > THRESHOLD_MB), since THRESHOLD_MB takes
+# precedence there (being smaller).
 THRESHOLD_RELATIVE = 0.15
 
+# DESIRED_FREE_DISK_SPACE is percentage of a disk to strive to keep available
+# (e.g. by trimming the isolate cache appropriately).
+#
+# If disk_size * DESIRED_FREE_DISK_SPACE is lower than self-quarantine threshold
+# (defined above), the self-quarantine threshold will be used instead. Will also
+# always add 250 MiB as an additional slack space for logs, temporary files etc.
+#
+# The goal is to always have some percentage of the disk available above
+# the self-quarantine threshold.
+#
+# The default values in practice mean:
+#   * For large disks (>27GB): try to keep 5%+250MB of disk free,
+#     self-quarantine if the free disk space is below 4GB.
+#   * For small disks (<27GB): try to keep 15%+250MB of disk free,
+#     self-quarantine if the free disk space is below 15%.
+DESIRED_FREE_DISK_SPACE = 0.05
+
 
 # https://cloud.google.com/compute/pricing#machinetype
 GCE_MACHINE_COST_HOUR_US = {
   u'n1-standard-1': 0.050,
   u'n1-standard-2': 0.100,
   u'n1-standard-4': 0.200,
   u'n1-standard-8': 0.400,
   u'n1-standard-16': 0.800,
   u'n1-standard-32': 1.600,
   u'f1-micro': 0.008,
@@ skipping 305 matching lines @@
 @tools.cached
 def get_min_free_space(path):
   """Returns the minimum free space (in MiB) required to not self-quarantine.
 
   Returns 0 when disabled.
   """
   return min(THRESHOLD_MB or 0, get_disk_size(path) * THRESHOLD_RELATIVE or 0)
 
 
 @tools.cached
+def get_desired_free_space(path):
+  """Returns the disk space (in MiB) to strive to keep available."""
+  mb = get_disk_size(path) * DESIRED_FREE_DISK_SPACE  # this can be 0
+  threshold = get_min_free_space(path)
+  if mb < threshold:
+    mb = threshold
+  return mb + 250
+
+
+@tools.cached
 def get_audio():
   """Returns the active audio card(s)."""
   # There's a risk that an audio card may "appear", which may be especially true
   # on OSX when an audio cable is plugged in.
   if sys.platform == 'darwin':
     return platforms.osx.get_audio()
   elif sys.platform == 'linux2':
     return platforms.linux.get_audio()
   elif sys.platform == 'win32':
     return platforms.win.get_audio()
@@ skipping 856 matching lines @@
 
 
 def trim_rolled_log(name):
   try:
     for item in glob.iglob('%s.??' % name):
       os.remove(item)
     for item in glob.iglob('%s.???' % name):
       os.remove(item)
   except Exception as e:
     logging.exception('trim_rolled_log(%s) failed: %s', name, e)