Teach build.py to cross-compile go-based packages.

build/test_packages.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Tests generated CIPD packages.
 
 Supposed to be invoked after build.py has run. Uses packages from out/*.cipd and
 tests from tests/*.py.
 
-Assumes cipd client is built in ../go/bin/cipd (true after build.py has run).
+Assumes cipd client is built in out/.cipd_client/cipd_* (true after build.py has
+run).
 """
 
 import argparse
 import glob
 import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
@@ skipping 23 matching lines @@
 def get_docstring(test_script):
   """Hacky way to grab a first line of a module docstring using regexps."""
   with open(test_script, 'rt') as f:
     text = f.read()
   m = re.match(r'^.*"""(.*?)"""', text, re.DOTALL)
   if not m:
     return None
   return m.group(1).strip().splitlines()[0]
 
 
+def find_cipd_client(out_dir):
+  """Returns path to cipd client built by build.py.
+
+  See build_cipd_client in build.py. It puts cipd client into
+  '<out_dir>/.cipd_client/cipd_<digest>' and there's only one such file there.
+
+  Prints error message and returns None if the file cannot be found.
+  """
+  out_dir = os.path.join(out_dir, '.cipd_client')
+  files = [f for f in os.listdir(out_dir) if f.startswith('cipd_')]
+  if not files:
+    print >> sys.stderr, 'Cannot find CIPD client in %s' % out_dir
+    return None
+  if len(files) != 1:
+    print >> sys.stderr, (
+        'There should be only one cipd client binary in %s, found %s' %
+        (out_dir, files))
+    return None
+  cipd_client = os.path.join(out_dir, files[0])
+  if not os.access(cipd_client, os.X_OK):
+    print >> sys.stderr, 'CIPD client at %s is not runnable'
+    return None
+  return cipd_client
+
+
 def run_test(cipd_client, package, work_dir, test_script):
   """Extracts a package to a dir and runs test_script with cwd == work_dir."""
   print_title('Deploying %s' % os.path.basename(package))
-  if not os.access(cipd_client, os.X_OK):
-    print >> sys.stderr, (
-        'CIPD client at %s doesn\'t exist or not runnable. Run build.py to '
-        'build it.' % cipd_client)
-    return 1
   cmd_line = ['cipd', 'pkg-deploy', '-root', work_dir, package]
   print ' '.join(cmd_line)
   if subprocess.call(args=cmd_line, executable=cipd_client):
     raise TestException('Failed to install %s, see logs' % package)
 
   print_title(get_docstring(test_script) or 'Running tests...')
   cmd_line = ['python', test_script]
   print '%s in %s' % (' '.join(cmd_line), work_dir)
   env = os.environ.copy()
   env.pop('PYTHONPATH', None)
   ret = subprocess.call(
       args=cmd_line, executable=sys.executable, env=env, cwd=work_dir)
   if ret:
     raise TestException('Non zero exit code (%d)' % ret)
 
 
 def run(
-    cipd_client,
     package_out_dir,
     package_tests_dir,
     work_dir,
     packages):
   """Deployes build *.cipd package locally and runs tests against them.
 
   Used to verify the packaged code works when installed as CIPD package, it is
   important for infra_python package that has non-trivial structure.
 
   Args:
-    cipd_client: path to cipd client executable.
     package_out_dir: where to search for built packages.
     work_dir: where to install/update packages into.
     packages: names of *.cipd files in package_out_dir or [] for all.
 
   Returns:
     0 on success, 1 or error.
   """
   # Discover what to test.
   paths = []
   if not packages:
-    paths = glob.glob(os.path.join(package_out_dir, '*.cipd'))

[Vadim Sh., 2016/06/25 03:51:04]

this method no longer works, since package_out_dir now may contain packages for
some other platform that the host can't test (since it can't run code in them).

+    # Enumerate all known tests in tests/*.py and filter them based on
+    # availability of corresponding *.cipd package in package_out_dir. It will
+    # skip any cross-compiled packages, since they have additional '+<platform>'
+    # suffix in the package file name.
+    for test in os.listdir(package_tests_dir):
+      if not test.endswith('.py'):
+        continue
+      pkg_file = os.path.join(
+          package_out_dir, os.path.splitext(test)[0] + '.cipd')
+      if os.path.exists(pkg_file):
+        paths.append(pkg_file)
   else:
     for name in packages:
       abs_path = os.path.join(package_out_dir, name)
       if not os.path.isfile(abs_path):
         raise TestException('No such package file: %s' % name)
       paths.append(abs_path)
   paths = sorted(paths)
   if not paths:
     print 'Nothing to test.'
     return 0
 
-  # Run all tests sequentially. There're like 2 of them tops.
+  cipd_client = find_cipd_client(package_out_dir)
+  if not cipd_client:
+    return 1
+
+  # Run all tests sequentially. Most of the are extra fast.
   nuke_temp = False
   if not work_dir:
     work_dir = tempfile.mkdtemp(suffix='cipd_test')
     nuke_temp = True
   work_dir = os.path.abspath(work_dir)
   try:
     fail = False
     for path in paths:
       name = os.path.splitext(os.path.basename(path))[0]
       test_script = os.path.join(package_tests_dir, '%s.py' % name)
@@ skipping 17 matching lines @@
   finally:
     if nuke_temp:
       try:
         shutil.rmtree(work_dir, ignore_errors=True)
       except OSError as exc:
         print >> sys.stderr, 'Failed to delete %s: %s' % (work_dir, exc)
 
 
 def main(
     args,
-    go_workspace=os.path.join(ROOT, 'go'),
     package_out_dir=os.path.join(ROOT, 'build', 'out'),
     package_tests_dir=os.path.join(ROOT, 'build', 'tests')):
   parser = argparse.ArgumentParser(description='Tests infra CIPD packages')
   parser.add_argument(
       'packages', metavar='NAME', type=str, nargs='*',
-      help='name of a build package file in build/out/* to deploy and test')
+      help='name of a built package file in build/out/* to deploy and test')
   parser.add_argument(
       '--work-dir', metavar='DIR', dest='work_dir',
       help='directory to deploy packages into (temporary dir by default)')
   args = parser.parse_args(args)
   return run(
-      os.path.join(go_workspace, 'bin', 'cipd' + EXE_SUFFIX),
       package_out_dir,
       package_tests_dir,
       args.work_dir,
       [n + '.cipd' if not n.endswith('.cipd') else n for n in args.packages])
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))