Add ability to store the manifest in the manifest directory.

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import heapq
 import re
 import optparse
 import os
 import shutil
 import sys
 
 import cbuildbot_comm
 from cbuildbot_config import config
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
 from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
                             Warning)
 
 _DEFAULT_RETRIES = 3
 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
+PUBLIC_OVERLAY = '%(buildroot)s/src/third_party/chromiumos-overlay'
+PRIVATE_OVERLAY = '%(buildroot)s/src/private-overlays/chromeos-overlay'
 
 # Currently, both the full buildbot and the preflight buildbot store their
 # data in a variable named PORTAGE_BINHOST, but they're in different files.
 # We're planning on joining the two files soon and renaming the full binhost
 # to FULL_BINHOST.
 _FULL_BINHOST = 'PORTAGE_BINHOST'
 _PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
 
 # ======================== Utility functions ================================
 
@@ skipping 31 matching lines @@
       RunCommand(['repo', '--trace', 'sync'], cwd=buildroot)
       retries = 0
     except:
       retries -= 1
       if retries > 0:
         Warning('CBUILDBOT -- Repo Sync Failed, retrying')
       else:
         Warning('CBUILDBOT -- Retries exhausted')
         raise
 
-  # Output manifest
-  RunCommand(['repo', 'manifest', '-r', '-o', '-'], cwd=buildroot)
-
 # =========================== Command Helpers =================================
 
 def _GetAllGitRepos(buildroot, debug=False):
   """Returns a list of tuples containing [git_repo, src_path]."""
   manifest_tuples = []
   # Gets all the git repos from a full repo manifest.
   repo_cmd = "repo manifest -o -".split()
   output = RunCommand(repo_cmd, cwd=buildroot, redirect_stdout=True,
                       redirect_stderr=True, print_cmd=debug)
 
@@ skipping 179 matching lines @@
   for mount_pt_str in mount_pts_in_buildroot.splitlines():
     mount_pt = mount_pt_str.rpartition(' type ')[0].partition(' on ')[2]
     RunCommand(['sudo', 'umount', '-l', mount_pt], error_ok=True)
 
 
 def _WipeOldOutput(buildroot):
   """Wipes out build output directories."""
   RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot)
 
 
+def _GetChromeOSVersion(buildroot):
+  """Returns the tuple version of the Chrome OS version of the buildroot."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  version_cmd = './chromeos_version.sh'
+  output = RunCommand(version_cmd, cwd=cwd, redirect_stdout=True,
+                      redirect_stderr=True)
+  version_re = re.compile('\s+CHROMEOS_VERSION_STRING='
+                          '(\d+)\.(\d+)\.(\d+)\.(\w+)')
+  for line in output.splitlines():
+    match = version_re.match(line)
+    if match:
+      return match.group(1), match.group(2), match.group(3), match.group(4)
+
+  raise Exception('Chrome OS version not found.')
+
+
+def _GetManifestPath(buildroot):
+  """Returns the relative path that a manifest should be saved into."""
+  version_tuple = _GetChromeOSVersion(buildroot)
+  (major, minor, branch, patch) = version_tuple
+  relative_path = os.path.join('.'.join([major, minor]),
+                               '%s.xml' % '.'.join(version_tuple))
+  return relative_path
+
+
 # =========================== Main Commands ===================================
 
 
 def _PreFlightRinse(buildroot, board, tracking_branch, overlays):
   """Cleans up any leftover state from previous runs."""
   _GitCleanup(buildroot, board, tracking_branch, overlays)
   _CleanUpMountPoints(buildroot)
   RunCommand(['sudo', 'killall', 'kvm'], error_ok=True)
 
 
 def _FullCheckout(buildroot, tracking_branch,
                   retries=_DEFAULT_RETRIES,
                   url='http://git.chromium.org/git/manifest'):
   """Performs a full checkout and clobbers any previous checkouts."""
   RunCommand(['sudo', 'rm', '-rf', buildroot])
   MakeDir(buildroot, parents=True)
   branch = tracking_branch.split('/');
   RunCommand(['repo', 'init', '-u',
              url, '-b',
              '%s' % branch[-1]], cwd=buildroot, input='\n\ny\n')
   RepoSync(buildroot, retries)
 
 
 def _IncrementalCheckout(buildroot, retries=_DEFAULT_RETRIES):
   """Performs a checkout without clobbering previous checkout."""
   RepoSync(buildroot, retries)
 
 
+def _DumpManifest(buildroot, url):
+  """Stores the manifest in the public | private overlay depending on url."""
+  public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot}

[scottz, 2010/12/11 00:01:53]

since you use {'buildroot': buildroot] in so many places why not set it in a
variable? 

str_replace_dict = {'buildroot': buildroot} then put that everywhere.

Just a suggestion

[sosa, 2010/12/13 18:46:39]

Rather not since buildroot would have to be defined before that was defined and
so i'd have to pass that around everywhere.

On 2010/12/11 00:01:53, scottz wrote:

+  private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot}
+  if url.endswith('manifest-internal'):
+    overlay = PRIVATE_OVERLAY % {'buildroot': buildroot}
+  else:
+    overlay = PUBLIC_OVERLAY % {'buildroot': buildroot}
+
+  # Generate paths for manifests.
+  relative_path = _GetManifestPath(buildroot)
+  manifest_path = os.path.join(overlay, 'manifests', relative_path)
+  symlink_path = os.path.join(overlay, 'manifests', 'LATEST')
+  if not os.path.isdir(os.path.dirname(manifest_path)):
+    os.makedirs(os.path.dirname(manifest_path))
+
+  # Dump the manifest and create a symlink to it.
+  RunCommand(['repo', 'manifest', '-r', '-o', manifest_path], cwd=buildroot)
+  if os.path.exists(symlink_path):
+    os.unlink(symlink_path)
+
+  os.symlink(relative_path, symlink_path)
+
+  # Add it to git and print it to stderr.
+  RunCommand(['git', 'add', os.path.join('manifests', relative_path)],
+             cwd=overlay)
+  RunCommand(['git', 'add', os.path.join('manifests', 'LATEST')], cwd=overlay)
+  shutil.copy(manifest_path, '/dev/stderr')

[scottz, 2010/12/11 00:01:53]

Why are we doing this?

[sosa, 2010/12/13 18:46:39]

Moved out to a function that does standard open, read, close

+
+
 def _MakeChroot(buildroot):
   """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
 def _GetPortageEnvVar(buildroot, board, envvar):
   """Get a portage environment variable for the specified board, if any.
 
   buildroot: The root directory where the build occurs. Must be an absolute
@@ skipping 201 matching lines @@
   """Return the list of overlays to use for a given buildbot.
 
   Args:
     buildroot: The root directory where the build occurs. Must be an absolute
                path.
     overlays: A string describing which overlays you want.
               'private': Just the private overlay.
               'public': Just the public overlay.
               'both': Both the public and private overlays.
   """
-  public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
-  private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
+  public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot}
+  private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot}
   if overlays == 'private':
     paths = [private_overlay]
   elif overlays == 'public':
     paths = [public_overlay]
   elif overlays == 'both':
     paths = [public_overlay, private_overlay]
   else:
     Info('No overlays found.')
     paths = []
   return paths
@@ skipping 108 matching lines @@
         _IncrementalCheckout(buildroot)
 
     new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
     emptytree = (old_binhost and old_binhost != new_binhost)
 
     # Check that all overlays can be found.
     for path in rev_overlays:
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
+    _DumpManifest(buildroot, options.url)
+
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
     # Perform uprev.  If chrome_uprev is set, rev Chrome ebuilds.
     if options.chrome_rev:
       chrome_atom_to_build = _MarkChromeAsStable(buildroot, tracking_branch,
                                                  options.chrome_rev)
@@ skipping 50 matching lines @@
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
 
     raise
 
 
 if __name__ == '__main__':
     main()