Remove GYP build from bots and 'gclient runhooks'

buildbot/buildbot_standard.py

 #!/usr/bin/python
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Enable 'with' statements in Python 2.5
 from __future__ import with_statement
 
 import os.path
 import platform
 import re
 import shutil
 import subprocess
 import sys
 import time
 
 from buildbot_lib import (
-    BuildContext, BuildStatus, Command, EnsureDirectoryExists,
+    BuildContext, BuildStatus, Command, EnsureDirectoryExists, GNArch,
     ParseStandardCommandLine, RemoveDirectory, RemovePath,
     RemoveGypBuildDirectories, RemoveSconsBuildDirectories, RunBuild, SCons,
-    SetupLinuxEnvironment, SetupMacEnvironment, SetupWindowsEnvironment,
-    SetupAndroidEnvironment, Step, StepLink, StepText, TryToCleanContents,
+    SetupLinuxEnvironment, SetupWindowsEnvironment,
+    Step, StepLink, StepText, TryToCleanContents,
     RunningOnBuildbot)
 
 
 def SetupContextVars(context):
   # The branch is set to native_client on the main bots, on the trybots it's
   # set to ''.  Otherwise, we should assume a particular branch is being used.
   context['branch'] = os.environ.get('BUILDBOT_BRANCH', 'native_client')
   context['off_trunk'] = context['branch'] not in ['native_client', '']
 
 
@@ skipping 38 matching lines @@
   ])
   # Copy html.
   Command(context, [
       sys.executable, gsutil,
       'cp', '-R', '-a', 'public-read',
       'html', gs_path,
   ], cwd=cov_dir)
   print '@@@STEP_LINK@view@%s@@@' % link_url
 
 
-def CommandGypBuild(context):
-  # Do not use goma when inside a toolchain build, because the
-  # freshly-built NaCl compilers will never be available via goma.
-  # This sacrifices the benefits of goma for building the trusted
-  # code too, but it's not clear how to teach Gyp to use goma for
-  # some compilers and not others.
-  use_goma = (RunningOnBuildbot() and
-              not context['no_goma'] and
-              not context['inside_toolchain'])
-
-  if use_goma:
-    # Since this is for buildbot, it should not be good to use the result
-    # generated by the different version compiler.
-    os.environ['GOMA_HERMETIC'] = 'fallback'
-  else:
-    os.environ['GOMA_DISABLED'] = '1'
-
-  runtest_py = os.environ.get('RUNTEST')
-  alt_runtest_py = '/b/build/scripts/slave/runtest.py'
-  if runtest_py is None and os.path.exists(alt_runtest_py):
-    runtest_py = alt_runtest_py
-
-  # TODO(bradnelson): Figure out why win64 trybots can't upload goma logs.
-  buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
-  excluded_os = False
-  for name in ['win64', 'vista', 'win7-64', 'win8-64']:
-    if name in buildername:
-      excluded_os = True
-
-  if runtest_py is None or excluded_os:
-    # Fallback to direct goma + ninja if not run on bots.
-    try:
-      if use_goma:
-        Command(context, cmd=[
-            sys.executable, '/b/build/goma/goma_ctl.py', 'restart'])
-      cmd = ['ninja', '-v', '-k', '0', '-C', '../out/' + context['gyp_mode']]
-      if use_goma:
-        cmd += ['-j50']
-      Command(context, cmd=cmd)
-    finally:
-      if use_goma:
-        Command(context, cmd=[
-            sys.executable, '/b/build/goma/goma_ctl.py', 'stop'])
-  else:
-    # Infer the location of compile.py from runtest.py.
-    compile_py = os.path.join(os.path.dirname(runtest_py), 'compile.py')
-    cmd = [sys.executable, compile_py, '--target', context['gyp_mode'],
-           '--src-dir', '../', '--build-tool', 'ninja',
-           '--ninja-ensure-up-to-date']
-    if use_goma:
-      cmd += ['--compiler', 'goma']
-      cmd += ['--goma-dir', '/b/build/goma']
-    # Verbose and but stop on fail.
-    cmd += ['--', '-v', '-k', '0']
-    Command(context, cmd=cmd)
-
-
-def CommandGypGenerate(context):
-  Command(
-          context,
-          cmd=[sys.executable, 'native_client/build/gyp_nacl'],
-          cwd='..')
-
-
 def CommandGclientRunhooks(context):
   if context.Windows():
     gclient = 'gclient.bat'
   else:
     gclient = 'gclient'
   print 'Running gclient runhooks...'
-  print 'GYP_CROSSCOMPILE=' + context.GetEnv('GYP_CROSSCOMPILE', '')
-  print 'GYP_GENERATORS=' + context.GetEnv('GYP_GENERATORS', '')
-  print 'GYP_MSVS_VERSION=' + context.GetEnv('GYP_MSVS_VERSION', '')
-  print 'GYP_DEFINES=' + context.GetEnv('GYP_DEFINES', '')
   Command(context, cmd=[gclient, 'runhooks', '--force'])
 
 
 def DoGNBuild(status, context, force_clang=False, force_arch=None):
+  if context['no_gn']:
+    return False
+
   use_clang = force_clang or context['clang']
 
   # Linux builds (or cross-builds) for every target.  Mac builds for
   # x86-32 and x86-64, and can build untrusted code for others.
   if context.Windows() and context['arch'] != '64':
     # The GN scripts for MSVC barf for a target_cpu other than x86 or x64
     # even if we only try to build the untrusted code.  Windows does build
     # for both x86-32 and x86-64 targets, but the GN Windows MSVC toolchain
     # scripts only support x86-64 hosts--and the Windows build of Clang
     # only has x86-64 binaries--while NaCl's x86-32 testing bots have to be
     # actual x86-32 hosts.
     return False
 
   if force_arch is not None:
     arch = force_arch
   else:
     arch = context['arch']
 
+  if context.Linux():
+    # The Linux build uses a sysroot.  'gclient runhooks' installs this
+    # for the default architecture, but this might be a cross-build that
+    # gclient didn't know was going to be done.  The script completes
+    # quickly when it's redundant with a previous run.
+    with Step('update_sysroot', status):
+      sysroot_arch = {'arm': 'arm',
+                      '32': 'i386',
+                      '64': 'amd64',
+                      'mips32': 'mips'}[arch]
+      Command(context, cmd=[sys.executable,
+                            '../build/linux/sysroot_scripts/install-sysroot.py',
+                            '--arch=' + sysroot_arch])
+
   out_suffix = '_' + arch
   if force_clang:
     out_suffix += '_clang'
   gn_out = '../out' + out_suffix
 
   def BoolFlag(cond):
     return 'true' if cond else 'false'
 
   gn_newlib = BoolFlag(not context['use_glibc'])
   gn_glibc = BoolFlag(context['use_glibc'])
-
-  gn_arch_name = {
-      'arm': 'arm',
-      '32': 'x86',
-      '64': 'x64'
-      }[arch]
+  gn_arch_name = GNArch(arch)
 
   gn_gen_args = [
       # The Chromium GN definitions might default enable_nacl to false
       # in some circumstances, but various BUILD.gn files involved in
       # the standalone NaCl build assume enable_nacl==true.
       'enable_nacl=true',
       'target_cpu="%s"' % gn_arch_name,
       'is_debug=' + context['gn_is_debug'],
       'use_gcc_glibc=' + gn_glibc,
       'use_clang_newlib=' + gn_newlib,
@@ skipping 175 matching lines @@
     if context.Mac():
       subprocess.call(
           "find /var/folders -name '.org.chromium.*' -exec rm -rfv '{}' ';'",
           shell=True)
       subprocess.call(
           "find /var/folders -name '.com.google.Chrome*' -exec rm -rfv '{}' ';'",
           shell=True)
 
   # Skip over hooks when run inside the toolchain build because
   # package_version would overwrite the toolchain build.
-  if inside_toolchain:
-    with Step('gyp_generate_only', status):
-      CommandGypGenerate(context)
-  else:
+  if not inside_toolchain:
     with Step('gclient_runhooks', status):
       CommandGclientRunhooks(context)
 
   # Always update Clang.  On Linux and Mac, it's the default for the GN build.
   # On Windows, we do a second Clang GN build.
   with Step('update_clang', status):
     Command(context, cmd=[sys.executable, '../tools/clang/scripts/update.py'])
 
   # Make sure our GN build is working.
   using_gn = DoGNBuild(status, context)
@@ skipping 40 matching lines @@
       ValidatorTest(
           context, 'x86-64',
           'scons-out/opt-linux-x86-64/staging/ncval_new')
 
     return
 
   # Run checkdeps script to vet #includes.
   with Step('checkdeps', status):
     Command(context, cmd=[sys.executable, 'tools/checkdeps/checkdeps.py'])
 
-  # Make sure our Gyp build is working.
-  if not context['no_gyp']:
-    with Step('gyp_compile', status):
-      CommandGypBuild(context)
-
   # On a subset of Linux builds, build Breakpad tools for testing.
   if context['use_breakpad_tools']:
     with Step('breakpad configure', status):
       Command(context, cmd=['mkdir', '-p', 'breakpad-out'])
       Command(context, cwd='breakpad-out',
               cmd=['bash', '../../breakpad/configure',
                    'CXXFLAGS=-I../..'])  # For third_party/lss
     with Step('breakpad make', status):
       Command(context, cmd=['make', '-j%d' % context['max_jobs'],
                             # This avoids a broken dependency on
@@ skipping 57 matching lines @@
 
 def Main():
   # TODO(ncbray) make buildbot scripts composable to support toolchain use case.
   context = BuildContext()
   status = BuildStatus(context)
   ParseStandardCommandLine(context)
   SetupContextVars(context)
   if context.Windows():
     SetupWindowsEnvironment(context)
   elif context.Linux():
-    if context['android']:
-      SetupAndroidEnvironment(context)
-    else:
+    if not context['android']:
       SetupLinuxEnvironment(context)
   elif context.Mac():
-    SetupMacEnvironment(context)
+    # No setup to do for Mac.
+    pass
   else:
     raise Exception("Unsupported platform.")
   RunBuild(BuildScript, status)
 
 
 def TimedMain():
   start_time = time.time()
   try:
     Main()
   finally:
     time_taken = time.time() - start_time
     print 'RESULT BuildbotTime: total= %.3f minutes' % (time_taken / 60)
 
 
 if __name__ == '__main__':
   TimedMain()