Add GN (the build tool) to the chromium.linux buildbots and try jobs

scripts/slave/recipes/chromium_trybot.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 DEPS = [
   'chromium',
   'gclient',
   'itertools',
   'json',
   'path',
@@ skipping 243 matching lines @@
 
     yield api.chromium.runhooks(abort_on_failure=False, can_fail_build=False)
     if api.step_history.last_step().retcode != 0:
       yield (
         api.path.rmcontents('slave build directory', api.path['slave_build']),
         api.gclient.checkout(revert=False),
         api.rietveld.apply_issue(),
         api.chromium.runhooks(),
       )
 
+  # TODO(dpranke): crbug.com/353690. Remove the gn-specific steps from this
+  # recipe and stand up a dedicated GN bot when the GN steps take up enough
+  # resources to be worth it. For now, we run GN and generate files into a new
+  # Debug_gn / Release_gn dir, and then run a compile in that dir.
+  gn_build_config_dir = str(api.chromium.c.BUILD_CONFIG) + '_gn'
+  gn_output_arg = '//out/' + gn_build_config_dir
+  gn_output_dir = api.path['checkout'].join('out', gn_build_config_dir)
+  should_run_gn = api.properties.get('buildername') in ('linux_chromium',
+                                                        'linux_chromium_rel')
+
   test_spec = api.step_history['read test spec'].json.output
   test_spec = [s.encode('utf-8') for s in test_spec]
 
   tests = []
   tests.append(CheckdepsTest())
   tests.append(Deps2GitTest())
   for name in GTEST_TESTS + test_spec:
     tests.append(GTestTest(name))
   tests.append(NaclIntegrationTest())
 
   compile_targets = list(api.itertools.chain(
                              *[t.compile_targets() for t in tests]))
   yield api.chromium.compile(compile_targets,
                              name='compile (with patch)',
                              abort_on_failure=False,
                              can_fail_build=False)
-  if api.step_history['compile (with patch)'].retcode != 0:
+  if should_run_gn:
+    yield api.chromium.run_gn(gn_output_arg)
+    yield api.chromium.compile_with_ninja('compile (gn with patch)',
+                                          gn_output_dir)
+
+  if ((api.step_history['compile (with patch)'].retcode != 0) or
+      (should_run_gn and
+       api.step_history['compile (gn with patch)'].retcode != 0)):

[Paweł Hajdan Jr., 2014/03/28 09:45:04]

For the fallback to be useful please mark compile_with_ninja above and run_gn
with abort_on_failure=False, can_fail_build=False.

This can actually get pretty complex, and so if you want to land this patch I
strongly recommend extracting the trybot recipe changes to a separate CL. In
fact, if you just create a separate gn trybot recipe it can make it much easier.

[Dirk Pranke, 2014/03/28 15:59:43]

Good catch, will do.
I started out this whole effort by trying to create separate CLs for trybots vs.
buildbots, and the figuring out what was changing where was too complicated
(none of us could keep track of which feedback was in which CL :).

Case in point: I can't create a separate gn trybot recipe, because we don't have
separate GN trybots, though I had hoped to do that at first. The changes *have*
to be in this file because we're changing the existing chromium trybot behavior.

As soon as we have separate standalone GN trybots, I will create a separate
recipe.

At any rate, changing the above two lines is simple enough. Was there anything
else keeping this from landing?

     # Only use LKCR when compile fails. Note that requested specific revision
     # can still override this.
     api.gclient.set_config('chromium_lkcr')
 
     # Since we're likely to switch to an earlier revision, revert the patch,
     # sync with the new config, and apply issue again.
     yield api.gclient.checkout(revert=True)
     yield api.rietveld.apply_issue()
 
     yield api.chromium.compile(compile_targets,
                                name='compile (with patch, lkcr, clobber)',
                                force_clobber=True,
                                abort_on_failure=False,
                                can_fail_build=False)
     if api.step_history['compile (with patch, lkcr, clobber)'].retcode != 0:
       yield (
         api.path.rmcontents('slave build directory', api.path['slave_build']),
         api.gclient.checkout(revert=False),
         api.rietveld.apply_issue(),
         api.chromium.runhooks(),
         api.chromium.compile(compile_targets,
                              name='compile (with patch, lkcr, clobber, nuke)',
                              force_clobber=True)
       )
 
+    if should_run_gn:
+      yield api.path.rmcontents('slave gn build directory', gn_output_dir)
+      yield api.chromium.run_gn(gn_output_arg)
+      yield api.chromium.compile_with_ninja(
+          'compile (gn with patch, lkcr, clobber)', gn_output_dir)
+
   # Do not run tests if the build is already in a failed state.
   if api.step_history.failed:
     return
 
   if recipe_config['compile_only']:
     return
 
-  # TODO(phajdan.jr): Make it possible to retry telemtry tests (add JSON).
+  # TODO(dpranke): crbug.com/353690. It would be good to run gn_unittests
+  # out of the gn build dir, but we can't use runtest()
+  # because of the different output directory; this means
+  # we don't get annotations and don't get retry of the tests for free :( .
+
+  # TODO(phajdan.jr): Make it possible to retry telemetry tests (add JSON).
   yield (
     api.chromium.run_telemetry_unittests(),
     api.chromium.run_telemetry_perf_unittests(),
   )
 
   def deapply_patch_fn(failing_tests):
     yield (
       api.gclient.revert(),
       api.chromium.runhooks(),
     )
@@ skipping 116 matching lines @@
   )
 
   yield (
     api.test('compile_failure_linux') +
     props() +
     api.platform.name('linux') +
     api.step_data('compile (with patch)', retcode=1) +
     api.step_data('compile (with patch, lkcr, clobber)', retcode=1) +
     api.step_data('compile (with patch, lkcr, clobber, nuke)', retcode=1)
   )
+
+  # TODO(dpranke): crbug.com/353690.
+  # Remove this when we make GN a standalone recipe.
+  yield (
+    api.test('unittest_should_run_gn') +
+    api.properties.tryserver(buildername='linux_chromium',
+                             build_config='Debug') +
+    api.platform.name('linux') +
+    api.step_data('compile (gn with patch)')
+  )
+
+  yield (
+    api.test('unittest_should_run_gn_compile_failure') +
+    api.properties.tryserver(buildername='linux_chromium',
+                             build_config='Debug') +
+    api.platform.name('linux') +
+    api.step_data('compile (gn with patch)', retcode=1)
+  )