infra/publish_tarball: move export_tarball.py to recipes

scripts/slave/recipe_modules/chromium/resources/export_tarball.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This tool creates a tarball with all the sources, but without .svn directories.
+
+It can also remove files which are not strictly required for build, so that
+the resulting tarball can be reasonably small (last time it was ~110 MB).
+
+Example usage:
+
+export_tarball.py /foo/bar
+
+The above will create file /foo/bar.tar.bz2.
+"""
+
+import optparse
+import os
+import subprocess
+import sys
+import tarfile
+
+
+NONESSENTIAL_DIRS = (
+    'breakpad/src/processor/testdata',
+    'chrome/browser/resources/tracing/tests',
+    'chrome/common/extensions/docs',
+    'chrome/tools/test/reference_build',
+    'courgette/testdata',
+    'data',
+    'native_client/src/trusted/service_runtime/testdata',
+    'src/chrome/test/data',
+    'o3d/documentation',
+    'o3d/samples',
+    'o3d/tests',
+    'ppapi/examples',
+    'ppapi/native_client/tests',
+    'third_party/angle/samples/gles2_book',
+    'third_party/findbugs',
+    'third_party/hunspell_dictionaries',
+    'third_party/hunspell/tests',
+    'third_party/lighttpd',
+    'third_party/sqlite/src/test',
+    'third_party/sqlite/test',
+    'third_party/vc_80',
+    'third_party/xdg-utils/tests',
+    'third_party/yasm/source/patched-yasm/modules/arch/x86/tests',
+    'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests',
+    'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests',
+    'third_party/WebKit/LayoutTests',
+    'third_party/WebKit/Source/JavaScriptCore/tests',
+    'third_party/WebKit/Source/WebCore/ChangeLog',
+    'third_party/WebKit/Source/WebKit2',
+    'third_party/WebKit/Tools/Scripts',
+    'tools/gyp/test',
+    'v8/test',
+    'webkit/data/layout_tests',
+    'webkit/tools/test/reference_build',
+)
+
+TESTDIRS = (
+    'chrome/test/data',
+    'content/test/data',
+    'media/test/data',
+    'net/data',
+)
+
+
+def GetSourceDirectory():
+  return os.path.realpath(
+    os.path.join(os.path.dirname(__file__), '..', '..', '..', 'src'))
+
+
+# Workaround lack of the exclude parameter in add method in python-2.4.
+# TODO(phajdan.jr): remove the workaround when it's not needed on the bot.

[pgervais, 2015/04/24 18:05:05]

Maybe it's time. The oldest Python we have is 2.6, correct?

[Paweł Hajdan Jr., 2015/04/24 18:15:39]

We should be on 2.7 everywhere. I plan to look into this (and other cleanups of
this file), it just seems safer to copy it as-is for now.

+class MyTarFile(tarfile.TarFile):
+  def set_remove_nonessential_files(self, remove):
+    self.__remove_nonessential_files = remove
+
+  def set_verbose(self, verbose):
+    self.__verbose = verbose
+
+  def __report_skipped(self, name):
+    if self.__verbose:
+      print 'D\t%s' % name
+
+  def __report_added(self, name):
+    if self.__verbose:
+      print 'A\t%s' % name
+
+  def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
+    head, tail = os.path.split(name)
+    if tail in ('.svn', '.git'):
+      self.__report_skipped(name)
+      return
+
+    if self.__remove_nonessential_files:
+      # WebKit change logs take quite a lot of space. This saves ~10 MB
+      # in a bzip2-compressed tarball.
+      if 'ChangeLog' in name:
+        self.__report_skipped(name)
+        return
+
+      # Remove contents of non-essential directories, but preserve gyp files,
+      # so that build/gyp_chromium can work.
+      for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS):
+        dir_path = os.path.join(GetSourceDirectory(), nonessential_dir)
+        if (name.startswith(dir_path) and
+            os.path.isfile(name) and
+            'gyp' not in name):
+          self.__report_skipped(name)
+          return
+
+    self.__report_added(name)
+    tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive)
+
+
+def main(argv):
+  parser = optparse.OptionParser()
+  parser.add_option("--basename")
+  parser.add_option("--remove-nonessential-files",
+                    dest="remove_nonessential_files",
+                    action="store_true", default=False)
+  parser.add_option("--test-data", action="store_true")
+  # TODO(phajdan.jr): Remove --xz option when it's not needed for compatibility.
+  parser.add_option("--xz", action="store_true")
+  parser.add_option("--verbose", action="store_true", default=False)
+  parser.add_option("--progress", action="store_true", default=False)
+
+  options, args = parser.parse_args(argv)
+
+  if len(args) != 1:
+    print 'You must provide only one argument: output file name'
+    print '(without .tar.xz extension).'
+    return 1
+
+  if not os.path.exists(GetSourceDirectory()):
+    print 'Cannot find the src directory ' + GetSourceDirectory()
+    return 1
+
+  # These two commands are from src/DEPS; please keep them in sync.

[pgervais, 2015/04/24 18:05:05]

I guess there is a similar comment in src/DEPS. Could you update it to reflect
the new location of this file? (or add it if it doesn't exist)

[Paweł Hajdan Jr., 2015/04/24 18:15:39]

I uploaded https://codereview.chromium.org/1105963002

The idea is this is only used for distro tarballs, and I don't want to burden
chrome devs with updating this, especially that now it'd be in a different repo
they're not familiar with.

[pgervais, 2015/04/24 18:32:22]

Keeping two things in sync in two different repos is a recipe for disaster.
Can't we find a way to break this dependency for real?

+  if subprocess.call(['python', 'build/util/lastchange.py', '-o',
+                      'build/util/LASTCHANGE'], cwd=GetSourceDirectory()) != 0:
+    print 'Could not run build/util/lastchange.py to update LASTCHANGE.'
+    return 1
+  if subprocess.call(['python', 'build/util/lastchange.py', '-s',
+                      'third_party/WebKit', '-o',
+                      'build/util/LASTCHANGE.blink'],
+                     cwd=GetSourceDirectory()) != 0:
+    print 'Could not run build/util/lastchange.py to update LASTCHANGE.blink.'
+    return 1
+
+  output_fullname = args[0] + '.tar'
+  output_basename = options.basename or os.path.basename(args[0])
+
+  archive = MyTarFile.open(output_fullname, 'w')
+  archive.set_remove_nonessential_files(options.remove_nonessential_files)
+  archive.set_verbose(options.verbose)
+  try:
+    if options.test_data:
+      for directory in TESTDIRS:
+        archive.add(os.path.join(GetSourceDirectory(), directory),
+                    arcname=os.path.join(output_basename, directory))
+    else:
+      archive.add(GetSourceDirectory(), arcname=output_basename)
+  finally:
+    archive.close()
+
+  if options.progress:
+    sys.stdout.flush()
+    pv = subprocess.Popen(
+        ['pv', '--force', output_fullname],
+        stdout=subprocess.PIPE,
+        stderr=sys.stdout)
+    with open(output_fullname + '.xz', 'w') as f:
+      rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f)
+    pv.wait()
+  else:
+    rc = subprocess.call(['xz', '-9', output_fullname])
+
+  if rc != 0:
+    print 'xz -9 failed!'
+    return 1
+
+  return 0
+
+
+if __name__ == "__main__":
+  sys.exit(main(sys.argv[1:]))