Move goma_teardown, goma_setup and determine_goma_jobs to goma_utils.py

scripts/slave/compile.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A tool to build chrome, executed by buildbot.
 
   When this is run, the current directory (cwd) should be the outer build
   directory (e.g., chrome-release/build/).
 
   For a list of command-line options, call this script with '--help'.
 """
 
-import multiprocessing
 import optparse
 import os
 import re
-import subprocess
 import sys
 
 from common import chromium_utils
 from slave import build_directory
 from slave import goma_utils
 
 # Define a bunch of directory paths (same as bot_update.py)
 CURRENT_DIR = os.path.abspath(os.getcwd())
 BUILDER_DIR = os.path.dirname(CURRENT_DIR)
 SLAVE_DIR = os.path.dirname(BUILDER_DIR)
 # GOMA_CACHE_DIR used for caching long-term data.
 DEFAULT_GOMA_CACHE_DIR = os.path.join(SLAVE_DIR, 'goma_cache')

[ukai, 2016/07/27 01:13:29]

can't move it to goma_utils?

[tikuta, 2016/07/27 02:33:30]

Done.

 
 # Path of the scripts/slave/ checkout on the slave, found by looking at the
 # current compile.py script's path's dirname().
 SLAVE_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
 # Path of the build/ checkout on the slave, found relative to the
 # scripts/slave/ directory.
 BUILD_DIR = os.path.dirname(os.path.dirname(SLAVE_SCRIPTS_DIR))
 
 
 class EchoDict(dict):
@@ skipping 24 matching lines @@
       fh = sys.stdout
     fh.write('Environment variables modified in compile.py:\n')
     for k in sorted(list(self.overrides)):
       if k in self:
         fh.write('  %s=%s\n' % (k, self[k]))
       else:
         fh.write('  %s (removed)\n' % k)
     fh.write('\n')
 
 
-# TODO(tikuta): move to goma_utils.py
-def goma_setup(options, env):
-  """Sets up goma if necessary.
-
-  If using the Goma compiler, first call goma_ctl  to ensure the proxy is
-  available, and returns (True, instance of cloudtail subprocess).
-  If it failed to start up compiler_proxy, modify options.compiler and
-  options.goma_dir and returns (False, None)
-
-  """
-  if options.compiler not in ('goma', 'goma-clang'):
-    # Unset goma_dir to make sure we'll not use goma.
-    options.goma_dir = None
-    return False, None
-
-  hostname = goma_utils.GetShortHostname()
-  # HACK(shinyak, yyanagisawa, goma): Windows NO_NACL_GOMA (crbug.com/390764)
-  # Building NaCl untrusted code using goma brings large performance
-  # improvement but it sometimes cause build failure by race condition.
-  # Let me enable goma build on goma canary buildslaves to confirm the issue
-  # has been fixed by a workaround.
-  # vm*-m4 are trybots. build*-m1 and vm*-m1 are all goma canary bots.
-  if hostname in ['build28-m1', 'build58-m1', 'vm191-m1', 'vm480-m1',
-                  'vm820-m1', 'vm821-m1', 'vm848-m1']:
-    env['NO_NACL_GOMA'] = 'false'
-
-  if options.goma_fail_fast:
-    # startup fails when initial ping failed.
-    env['GOMA_FAIL_FAST'] = 'true'
-  else:
-    # If a network error continues 30 minutes, compiler_proxy make the compile
-    # failed.  When people use goma, they expect using goma is faster than
-    # compile locally. If goma cannot guarantee that, let it make compile
-    # as error.
-    env['GOMA_ALLOWED_NETWORK_ERROR_DURATION'] = '1800'
-
-  # HACK(yyanagisawa): reduce GOMA_BURST_MAX_PROCS crbug.com/592306
-  # Recently, I sometimes see buildbot slave time out, one possibility I come
-  # up with is burst mode use up resource.
-  # Let me temporary set small values to GOMA_BURST_MAX_PROCS to confirm
-  # the possibility is true or false.
-  max_subprocs = '3'
-  max_heavy_subprocs = '1'
-  number_of_processors = 0
-  try:
-    number_of_processors = multiprocessing.cpu_count()
-  except NotImplementedError:
-    print 'cpu_count() is not implemented, using default value.'
-    number_of_processors = 1
-  if number_of_processors > 3:
-    max_subprocs = str(number_of_processors - 1)
-    max_heavy_subprocs = str(number_of_processors / 2)
-  env['GOMA_BURST_MAX_SUBPROCS'] = max_subprocs
-  env['GOMA_BURST_MAX_SUBPROCS_LOW'] = max_subprocs
-  env['GOMA_BURST_MAX_SUBPROCS_HEAVY'] = max_heavy_subprocs
-
-  # Caches CRLs in GOMA_CACHE_DIR.
-  # Since downloading CRLs is usually slow, caching them may improves
-  # compiler_proxy start time.
-  if not os.path.exists(options.goma_cache_dir):
-    os.mkdir(options.goma_cache_dir, 0700)
-  env['GOMA_CACHE_DIR'] = options.goma_cache_dir
-
-  # Enable DepsCache. DepsCache caches the list of files to send goma server.
-  # This will greatly improve build speed when cache is warmed.
-  # The cache file is stored in the target output directory.
-  env['GOMA_DEPS_CACHE_DIR'] = (
-      options.goma_deps_cache_dir or options.target_output_dir)
-
-  if not env.get('GOMA_HERMETIC'):
-    env['GOMA_HERMETIC'] = options.goma_hermetic
-  if options.goma_enable_remote_link:
-    env['GOMA_ENABLE_REMOTE_LINK'] = options.goma_enable_remote_link
-  if options.goma_store_local_run_output:
-    env['GOMA_STORE_LOCAL_RUN_OUTPUT'] = options.goma_store_local_run_output
-  if options.goma_enable_compiler_info_cache:
-    # Will be stored in GOMA_CACHE_DIR.
-    env['GOMA_COMPILER_INFO_CACHE_FILE'] = 'goma-compiler-info.cache'
-
-  if options.build_data_dir:
-    env['GOMA_DUMP_STATS_FILE'] = os.path.join(options.build_data_dir,
-                                               'goma_stats_proto')
-
-  # goma is requested.
-  goma_key = os.path.join(options.goma_dir, 'goma.key')
-  if os.path.exists(goma_key):
-    env['GOMA_API_KEY_FILE'] = goma_key
-  if options.goma_service_account_json_file:
-    env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = \
-        options.goma_service_account_json_file
-  if chromium_utils.IsWindows():
-    env['GOMA_RPC_EXTRA_PARAMS'] = '?win'
-  goma_start_command = ['restart']
-  goma_ctl_cmd = [sys.executable,
-                  os.path.join(options.goma_dir, 'goma_ctl.py')]
-  result = chromium_utils.RunCommand(goma_ctl_cmd + goma_start_command, env=env)
-  if not result:
-    # goma started sucessfully.
-    # Making cloudtail to upload the latest log.
-    # TODO(yyanagisawa): install cloudtail from CIPD.
-    cloudtail_path = '/opt/infra-tools/cloudtail'
-    if chromium_utils.IsWindows():
-      cloudtail_path = 'C:\\infra-tools\\cloudtail'
-    try:
-      cloudtail_proc = subprocess.Popen(
-          [cloudtail_path, 'tail', '--log-id', 'goma_compiler_proxy', '--path',
-           goma_utils.GetLatestGomaCompilerProxyInfo()])
-    except Exception as e:
-      print 'failed to invoke cloudtail: %s' % e
-      return True, None
-    return True, cloudtail_proc
-
-  if options.goma_jsonstatus:
-    chromium_utils.RunCommand(
-        goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env)
-    goma_utils.SendGomaTsMon(options.goma_jsonstatus, -1)
-
-  # Try to stop compiler_proxy so that it flushes logs and stores
-  # GomaStats.
-  if options.build_data_dir:
-    env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join(options.build_data_dir,
-                                                       'crash_report_id_file')
-  chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
-
-  override_gsutil = None
-  if options.gsutil_py_path:
-    override_gsutil = [sys.executable, options.gsutil_py_path]
-
-  # Upload compiler_proxy.INFO to investigate the reason of compiler_proxy
-  # start-up failure.
-  goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil)
-  # Upload GomaStats to make it monitored.
-  if env.get('GOMA_DUMP_STATS_FILE'):
-    goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'],
-                             env.get('GOMACTL_CRASH_REPORT_ID_FILE'),
-                             options.build_data_dir)
-
-  if options.goma_disable_local_fallback:
-    print 'error: failed to start goma; fallback has been disabled'
-    raise Exception('failed to start goma')
-
-  print 'warning: failed to start goma. falling back to non-goma'
-  # Drop goma from options.compiler
-  options.compiler = options.compiler.replace('goma-', '')
-  if options.compiler == 'goma':
-    options.compiler = None
-  # Reset options.goma_dir.
-  options.goma_dir = None
-  env['GOMA_DISABLED'] = '1'
-  return False, None
-
-
-# TODO(tikuta): move to goma_utils.py
-def goma_teardown(options, env, exit_status, cloudtail_proc):
-  """Tears down goma if necessary. """
-  if (options.compiler in ('goma', 'goma-clang') and
-      options.goma_dir):
-    override_gsutil = None
-    if options.gsutil_py_path:
-      override_gsutil = [sys.executable, options.gsutil_py_path]
-
-    # If goma compiler_proxy crashes during the build, there could be crash
-    # dump.
-    if options.build_data_dir:
-      env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join(options.build_data_dir,
-                                                         'crash_report_id_file')
-    goma_ctl_cmd = [sys.executable,
-                    os.path.join(options.goma_dir, 'goma_ctl.py')]
-    if options.goma_jsonstatus:
-      chromium_utils.RunCommand(
-          goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env)
-      goma_utils.SendGomaTsMon(options.goma_jsonstatus, exit_status)
-    # Always stop the proxy for now to allow in-place update.
-    chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
-    goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil)
-    if env.get('GOMA_DUMP_STATS_FILE'):
-      goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'],
-                               env.get('GOMACTL_CRASH_REPORT_ID_FILE'),
-                               options.build_data_dir)
-  if cloudtail_proc:
-    cloudtail_proc.terminate()
-    cloudtail_proc.wait()
-
-
 def maybe_set_official_build_envvars(options, env):
   if options.mode == 'google_chrome' or options.mode == 'official':
     env['CHROMIUM_BUILD'] = '_google_chrome'
 
   if options.mode == 'official':
     # Official builds are always Google Chrome.
     env['CHROME_BUILD_TYPE'] = '_official'
 
 
 class EnsureUpToDateFilter(chromium_utils.RunCommandFilter):
   """Filter for RunCommand that checks whether the output contains ninja's
   message for a no-op build."""
   def __init__(self):
     self.was_up_to_date = False
 
   def FilterLine(self, a_line):
     if 'ninja: no work to do.' in a_line:
       self.was_up_to_date = True
     return a_line
 
 
 def NeedEnvFileUpdateOnWin(env):

[ukai, 2016/07/27 01:13:29]

move it to goma_utils?

[tikuta, 2016/07/27 02:33:30]

Done.

   """Returns true if environment file need to be updated."""
   # Following GOMA_* are applied to compiler_proxy not gomacc,
   # you do not need to update environment files.
   ignore_envs = (
       'GOMA_API_KEY_FILE',
       'GOMA_DEPS_CACHE_DIR',
       'GOMA_HERMETIC',
       'GOMA_RPC_EXTRA_PARAMS',
       'GOMA_ALLOWED_NETWORK_ERROR_DURATION'
   )
@@ skipping 49 matching lines @@
         if not entry:
           continue
         key, value = entry.split('=', 1)
         env_in_file[key] = value
     env_in_file.update(env_to_store)
     with open(path, 'wb') as f:
       f.write(nul.join(['%s=%s' % (k, v) for k, v in env_in_file.iteritems()]))
       f.write(nul * 2)
 
 
-# TODO(tikuta): move to goma_utils
-def determine_goma_jobs():
-  # We would like to speed up build on Windows a bit, since it is slowest.
-  number_of_processors = 0
-  try:
-    number_of_processors = multiprocessing.cpu_count()
-  except NotImplementedError:
-    print 'cpu_count() is not implemented, using default value 50.'
-    return 50
-
-  assert number_of_processors > 0
-
-  # When goma is used, 10 * number_of_processors is basically good in
-  # various situations according to our measurement. Build speed won't
-  # be improved if -j is larger than that.
-  #
-  # Since Mac had process number limitation before, we had to set
-  # the upper limit to 50. Now that the process number limitation is 2000,
-  # so we would be able to use 10 * number_of_processors.
-  # For the safety, we'd like to set the upper limit to 200.
-  #
-  # Note that currently most try-bot build slaves have 8 processors.
-  if chromium_utils.IsMac() or chromium_utils.IsWindows():
-    return min(10 * number_of_processors, 200)
-
-  # For Linux, we also would like to use 10 * cpu. However, not sure
-  # backend resource is enough, so let me set Linux and Linux x64 builder
-  # only for now.
-  hostname = goma_utils.GetShortHostname()
-  if hostname in (
-      ['build14-m1', 'build48-m1'] +
-      # Also increasing cpus for v8/blink trybots.
-      ['build%d-m4' % x for x in xrange(45, 48)] +
-      # Also increasing cpus for LTO buildbots.
-      ['slave%d-c1' % x for x in [20, 33] + range(78, 108)]):
-    return min(10 * number_of_processors, 200)
-
-  return 50
-
-
 def main_ninja(options, args, env):
   """This function calls ninja.
 
   Args:
       options (Option): options for ninja command.
       args (str): extra args for ninja command.
       env (dict): Used when ninja command executes.
 
   Returns:
       int: ninja command exit status.
 
   """
 
   exit_status = -1
 
   try:
     print 'chdir to %s' % options.src_dir
     os.chdir(options.src_dir)
 
     command = [options.ninja_path, '-w', 'dupbuild=err',
                                    '-C', options.target_output_dir]
 
     # HACK(yyanagisawa): update environment files on |env| update.

[ukai, 2016/07/27 01:13:29]

might be goma specific (part of goma_setup)?

[tikuta, 2016/07/27 02:33:30]

Done.

     # For compiling on Windows, environment in environment files are used.
     # It means even if enviroment such as GOMA_DISABLED is updated in
     # compile.py, the update will be ignored.
     # We need to update environment files to reflect the update.
     if chromium_utils.IsWindows() and NeedEnvFileUpdateOnWin(env):
       print 'Updating environment.{x86,x64} files.'
       UpdateWindowsEnvironment(options.target_output_dir, env)
 
     if options.clobber:
       print 'Removing %s' % options.target_output_dir
@@ skipping 130 matching lines @@
   return options, args
 
 
 def real_main():
   options, args = get_parsed_options()
 
   # Prepare environment.
   env = EchoDict(os.environ)
 
   # start goma
-  goma_ready, goma_cloudtail = goma_setup(options, env)
+  goma_ready, goma_cloudtail = goma_utils.goma_setup(options, env)
 
   if not goma_ready:

[ukai, 2016/07/27 01:13:29]

if goma_ready:
  if options.goma_jobs is None:
     options.goma_jobs = ..
else:
  assert ..
  options.goma_jobs = ..
?

[tikuta, 2016/07/27 02:33:30]

Done.

     assert options.compiler not in ('goma', 'goma-clang')
     assert options.goma_dir is None
   elif options.goma_jobs is None:
-    options.goma_jobs = determine_goma_jobs()
+    options.goma_jobs = goma_utils.determine_goma_jobs()
 
   # build
   exit_status = main_ninja(options, args, env)
 
   # stop goma
-  goma_teardown(options, env, exit_status, goma_cloudtail)
+  goma_utils.goma_teardown(options, env, exit_status, goma_cloudtail)
 
   return exit_status
 
 
 if '__main__' == __name__:
   sys.exit(real_main())