Expose named caches as dimensions.

appengine/swarming/swarming_bot/api/os_utilities.py

+# coding: utf-8
 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """OS specific utility functions.
 
 Includes code:
 - to declare the current system this code is running under.
 - to run a command on user login.
 - to reboot the host.
@@ skipping 496 matching lines @@
   if sys.platform == 'darwin':
     return platforms.osx.get_uptime()
   if sys.platform == 'win32':
     return platforms.win.get_uptime()
   if sys.platform == 'cygwin':
     # Not important.
     return 0.
   return platforms.linux.get_uptime()
 
 
+def get_named_caches():
+  """Returns the list of named caches."""
+  # Strictly speaking, this is a layering violation. This data is managed by
+  # run_isolated.py but this is valuable to expose this as a Swarming bot
+  # dimensions so ¯\_(ツ)_/¯
+  #
+  # Assumptions:
+  # - ../__main__.py calls os.chdir(__file__)
+  # - ../bot_code/bot_main.py specifies
+  #   --named-cache-root os.path.join(botobj.base_dir, 'c') to run_isolated.
+  # - ../client/named_cache.py behavior
+  #
+  # A better implementation would require:
+  # - Access to bot.Bot instance to query bot.base_dir
+  # - Access to named_cache.py to load state.json
+  # - Access to --named-cache-root hardcoded in bot_main.py
+  #
+  # but hey, the following code is 4 lines...
+  d = os.path.join(os.getcwd(), u'c', u'named')

[nodir, 2017/05/31 16:52:06]

"named" symlinks are best effort. Better not to put them to the critical path.

Can we read c/state.json here instead? the code would be still simple

[M-A Ruel, 2017/05/31 20:45:32]

Then maybe we shouldn't create them. :/
Either we ensure the symlinks are good when cleanup() is called, either we drop
them, I prefer to not keep mostly-works paths.
Done.

[nodir, 2017/05/31 21:03:21]

we keep them for troopers and alike. My point is that it is not fatal to fail to
create a symlink, for example.

+  if not os.path.isdir(d):
+    return []
+  return [i for i in os.listdir(d)]
+
+
 class AuthenticatedHttpRequestFailure(Exception):
   pass
 
 
 def authenticated_http_request(service_account, *args, **kwargs):
   """Sends an OAuth2-authenticated HTTP request.
 
   Args:
     service_account: Service account to use. For GCE, the name of the service
       account, otherwise the path to the service account JSON file.
@@ skipping 315 matching lines @@
     u'cores': [unicode(get_num_processors())],
     u'cpu': [
       cpu_type,
       cpu_type + u'-' + cpu_bitness,
     ],
     u'gpu': get_gpu()[0],
     u'id': [get_hostname_short()],
     u'os': [os_name],
     u'pool': [u'default'],
   }
+  caches = get_named_caches()
+  if caches:
+    dimensions[u'caches'] = caches
   if u'avx2' in cpuinfo.get(u'flags', []):
     dimensions[u'cpu'].append(cpu_type + u'-' + cpu_bitness + u'-avx2')
   if any(u'avx512' in x for x in cpuinfo.get(u'flags', [])):
     dimensions[u'cpu'].append(cpu_type + u'-' + cpu_bitness + u'-avx512')
   if sys.platform == 'win32':
     dimensions[u'os'].extend(
         u'%s-%s' % (os_name, n) for n in platforms.win.get_os_version_names())
   else:
     dimensions[u'os'].append(u'%s-%s' % (os_name, get_os_version_number()))
   if u'none' not in dimensions[u'gpu']:
@@ skipping 342 matching lines @@
 
 
 def trim_rolled_log(name):
   try:
     for item in glob.iglob('%s.??' % name):
       os.remove(item)
     for item in glob.iglob('%s.???' % name):
       os.remove(item)
   except Exception as e:
     logging.exception('trim_rolled_log(%s) failed: %s', name, e)