Port chromium landmines script.

build/landmines.py

Index: build/landmines.py
diff --git a/build/landmines.py b/build/landmines.py
index bd1fb28f719c622a90888a90ca41815729c3b996..a35123afc528ebd13832d966593c3350254bedb6 100755
--- a/build/landmines.py
+++ b/build/landmines.py
@@ -4,10 +4,9 @@
 # found in the LICENSE file.
 
 """
-This script runs every build as a hook. If it detects that the build should
-be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
-various build scripts will then check for the presence of this file and clobber
-accordingly. The script will also emit the reasons for the clobber to stdout.
+This script runs every build as the first hook (See DEPS). If it detects that
+the build should be clobbered, it will delete the contents of the build
+directory.
 
 A landmine is tripped when a builder checks out a different revision, and the
 diff between the new landmines and the old ones is non-null. At this point, the
@@ -15,9 +14,13 @@ build is clobbered.
 """
 
 import difflib
+import errno
+import gyp_environment
 import logging
 import optparse
 import os
+import re
+import shutil
 import sys
 import subprocess
 import time
@@ -28,46 +31,118 @@ import landmine_utils
 SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
-def get_target_build_dir(build_tool, target):
+def get_build_dir(build_tool, is_iphone=False):
   """
   Returns output directory absolute path dependent on build and targets.
   Examples:
-    r'c:\b\build\slave\win\build\src\out\Release'
-    '/mnt/data/b/build/slave/linux/build/src/out/Debug'
-    '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
+    r'c:\b\build\slave\win\build\src\out'
+    '/mnt/data/b/build/slave/linux/build/src/out'
+    '/b/build/slave/ios_rel_device/build/src/xcodebuild'
 
   Keep this function in sync with tools/build/scripts/slave/compile.py
   """
   ret = None
   if build_tool == 'xcode':
-    ret = os.path.join(SRC_DIR, 'xcodebuild', target)
+    ret = os.path.join(SRC_DIR, 'xcodebuild')
   elif build_tool in ['make', 'ninja', 'ninja-ios']:  # TODO: Remove ninja-ios.
-    ret = os.path.join(SRC_DIR, 'out', target)
+    if 'CHROMIUM_OUT_DIR' in os.environ:
+      output_dir = os.environ.get('CHROMIUM_OUT_DIR').strip()
+      if not output_dir:
+        raise Error('CHROMIUM_OUT_DIR environment variable is set but blank!')
+    else:
+      output_dir = landmine_utils.gyp_generator_flags().get('output_dir', 'out')
+    ret = os.path.join(SRC_DIR, output_dir)
   elif build_tool in ['msvs', 'vs', 'ib']:

[Michael Achenbach, 2015/02/24 15:30:27]

This is different to chromium's landmines script which has no default anymore
for 'msvs', 'vs'.

-    ret = os.path.join(SRC_DIR, 'build', target)
+    ret = os.path.join(SRC_DIR, 'build')
   else:
     raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
   return os.path.abspath(ret)
 
 
-def set_up_landmines(target, new_landmines):
-  """Does the work of setting, planting, and triggering landmines."""
-  out_dir = get_target_build_dir(landmine_utils.builder(), target)
-
-  landmines_path = os.path.join(out_dir, '.landmines')
-  if not os.path.exists(out_dir):
+def extract_gn_build_commands(build_ninja_file):
+  """Extracts from a build.ninja the commands to run GN.
+
+  The commands to run GN are the gn rule and build.ninja build step at the
+  top of the build.ninja file. We want to keep these when deleting GN builds
+  since we want to preserve the command-line flags to GN.
+
+  On error, returns the empty string."""
+  result = ""
+  with open(build_ninja_file, 'r') as f:
+    # Read until the second blank line. The first thing GN writes to the file
+    # is the "rule gn" and the second is the section for "build build.ninja",
+    # separated by blank lines.
+    num_blank_lines = 0
+    while num_blank_lines < 2:
+      line = f.readline()
+      if len(line) == 0:
+        return ''  # Unexpected EOF.
+      result += line
+      if line[0] == '\n':
+        num_blank_lines = num_blank_lines + 1
+  return result
+
+def delete_build_dir(build_dir):
+  # GN writes a build.ninja.d file. Note that not all GN builds have args.gn.
+  build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d')
+  if not os.path.exists(build_ninja_d_file):
+    shutil.rmtree(build_dir)
     return
 
-  if not os.path.exists(landmines_path):
-    print "Landmines tracker didn't exists."
+  # GN builds aren't automatically regenerated when you sync. To avoid
+  # messing with the GN workflow, erase everything but the args file, and
+  # write a dummy build.ninja file that will automatically rerun GN the next
+  # time Ninja is run.
+  build_ninja_file = os.path.join(build_dir, 'build.ninja')
+  build_commands = extract_gn_build_commands(build_ninja_file)
+
+  try:
+    gn_args_file = os.path.join(build_dir, 'args.gn')
+    with open(gn_args_file, 'r') as f:
+      args_contents = f.read()
+  except IOError:
+    args_contents = ''
+
+  shutil.rmtree(build_dir)
+
+  # Put back the args file (if any).
+  os.mkdir(build_dir)
+  if args_contents != '':
+    with open(gn_args_file, 'w') as f:
+      f.write(args_contents)
+
+  # Write the build.ninja file sufficiently to regenerate itself.
+  with open(os.path.join(build_dir, 'build.ninja'), 'w') as f:
+    if build_commands != '':
+      f.write(build_commands)
+    else:
+      # Couldn't parse the build.ninja file, write a default thing.
+      f.write('''rule gn
+command = gn -q gen //out/%s/
+description = Regenerating ninja files
+
+build build.ninja: gn
+generator = 1
+depfile = build.ninja.d
+''' % (os.path.split(build_dir)[1]))
+
+  # Write a .d file for the build which references a nonexistant file. This
+  # will make Ninja always mark the build as dirty.
+  with open(build_ninja_d_file, 'w') as f:
+    f.write('build.ninja: nonexistant_file.gn\n')
+
+
+def clobber_if_necessary(new_landmines):
+  """Does the work of setting, planting, and triggering landmines."""
+  out_dir = get_build_dir(landmine_utils.builder())
+  landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
+  try:
+    os.makedirs(out_dir)
+  except OSError as e:
+    if e.errno == errno.EEXIST:
+      pass
 
-  # FIXME(machenbach): Clobber deletes the .landmines tracker. Difficult
-  # to know if we are right after a clobber or if it is first-time landmines
-  # deployment. Also, a landmine-triggered clobber right after a clobber is
-  # not possible. Different clobber methods for msvs, xcode and make all
-  # have different blacklists of files that are not deleted.
   if os.path.exists(landmines_path):
-    triggered = os.path.join(out_dir, '.landmines_triggered')
     with open(landmines_path, 'r') as f:
       old_landmines = f.readlines()
     if old_landmines != new_landmines:
@@ -75,14 +150,20 @@ def set_up_landmines(target, new_landmines):
       diff = difflib.unified_diff(old_landmines, new_landmines,
           fromfile='old_landmines', tofile='new_landmines',
           fromfiledate=old_date, tofiledate=time.ctime(), n=0)
-
-      with open(triggered, 'w') as f:
-        f.writelines(diff)
-      print "Setting landmine: %s" % triggered
-    elif os.path.exists(triggered):
-      # Remove false triggered landmines.
-      os.remove(triggered)
-      print "Removing landmine: %s" % triggered
+      sys.stdout.write('Clobbering due to:\n')
+      sys.stdout.writelines(diff)
+
+      # Clobber contents of build directory but not directory itself: some
+      # checkouts have the build directory mounted.
+      for f in os.listdir(out_dir):
+        path = os.path.join(out_dir, f)
+        # Soft version of chromium's clobber. Only delete directories not files
+        # as e.g. on windows the output dir is the build dir that shares some
+        # checked out files.
+        if os.path.isdir(path) and re.search(r"(?:[Rr]elease)|(?:[Dd]ebug)", f):

[Michael Achenbach, 2015/02/24 15:30:27]

This is different to chromium's landmines script. Only delete directories that
are called *release* or *debug*.

[jochen (gone - plz use gerrit), 2015/02/24 15:45:22]

what about optdebug for developer checkouts?

[Michael Achenbach, 2015/02/24 16:29:32]

re.search should return true for that as well. It'll e.g. delete also
"ia32.release", "Debug_x64", etc.

+          delete_build_dir(path)
+
+  # Save current set of landmines for next time.
   with open(landmines_path, 'w') as f:
     f.writelines(new_landmines)
 
@@ -123,14 +204,14 @@ def main():
   if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
     return 0
 
+  gyp_environment.set_environment()
+
   landmines = []
   for s in landmine_scripts:
     proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
     output, _ = proc.communicate()
     landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
-
-  for target in ('Debug', 'Release'):
-    set_up_landmines(target, landmines)
+  clobber_if_necessary(landmines)
 
   return 0