Add script for generating a source tarball and rules for bilding a Debian package

tools/create_tarball.py

+#!/usr/bin/env python
+#
+# Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+#
+
+# Script to build a tarball of the Dart source.
+#
+# The tarball includes all the source needed to build Dart. This
+# includes source in third_party. As part of creating the tarball the
+# files used to build Debian packages are copied to a top-level debian
+# directory. This makes it easy to build Debian packages from the
+# tarball.
+#
+# For building a Debian package renaming the tarball to follow the
+# Debian is needed.

[kustermann, 2014/02/06 15:28:04]

"to follow the Debian" sounds strange.

[Søren Gjesse, 2014/02/07 09:52:43]

Done.

+#
+#  $ mv dart-XXX.tar.gz dart_XXX.orig.tar.gz
+#  $ tar xf dart_XXX.orig.tar.gz
+#  $ cd dart_XXX
+#  $ debuild -us -uc
+
+import datetime
+import optparse
+import sys
+import tarfile
+import tempfile

[kustermann, 2014/02/06 15:28:04]

tempfile not used.

[Søren Gjesse, 2014/02/07 09:52:43]

Removed.

+import utils
+
+from os import listdir, makedirs, remove, rmdir

[kustermann, 2014/02/06 15:28:04]

remove, rmdir not used.

[Søren Gjesse, 2014/02/07 09:52:43]

Removed.

+from os.path import basename, dirname, join, realpath, exists, isdir, split

[kustermann, 2014/02/06 15:28:04]

basename, dirname, realpath, isdir unused.

[Søren Gjesse, 2014/02/07 09:52:43]

Removed.

+
+HOST_OS = utils.GuessOS()
+
+# TODO (16582): Remove this when the LICENSE file becomes part of
+# all checkouts.
+license = [

[kustermann, 2014/02/06 15:28:04]

Python supports multiline strings, why don't you do something like this?:

license = """
..........
..........
"""

[Søren Gjesse, 2014/02/07 09:52:43]

This will go away when https://codereview.chromium.org/156573003/ lands.

+  'This license applies to all parts of Dart that are not externally',
+  'maintained libraries. The external maintained libraries used by',
+  'Dart are:',
+  '',
+  '7-Zip - in third_party/7zip',
+  'JSCRE - in runtime/third_party/jscre',
+  'Ant - in third_party/apache_ant',
+  'args4j - in third_party/args4j',
+  'bzip2 - in third_party/bzip2',
+  'Commons IO - in third_party/commons-io',
+  'Commons Lang in third_party/commons-lang',
+  'dromaeo - in samples/third_party/dromaeo',
+  'Eclipse - in third_party/eclipse',
+  'gsutil - in third_party/gsutil',
+  'Guava - in third_party/guava',
+  'hamcrest - in third_party/hamcrest',
+  'Httplib2 - in samples/third_party/httplib2',
+  'JSON - in third_party/json',
+  'JUnit - in third_party/junit',
+  'Oauth - in samples/third_party/oauth2client',
+  'weberknecht - in third_party/weberknecht',
+  'fest - in third_party/fest',
+  'mockito - in third_party/mockito',
+  '',
+  'The libraries may have their own licenses; we recommend you read them,',
+  'as their terms may differ from the terms below.',
+  '',
+  'Copyright 2012, the Dart project authors. All rights reserved.',
+  'Redistribution and use in source and binary forms, with or without',
+  'modification, are permitted provided that the following conditions are',
+  'met:',
+  '    * Redistributions of source code must retain the above copyright',
+  '      notice, this list of conditions and the following disclaimer.',
+  '    * Redistributions in binary form must reproduce the above',
+  '      copyright notice, this list of conditions and the following',
+  '      disclaimer in the documentation and/or other materials provided',
+  '      with the distribution.',
+  '    * Neither the name of Google Inc. nor the names of its',
+  '      contributors may be used to endorse or promote products derived',
+  '      from this software without specific prior written permission.',
+  'THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS',
+  '"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT',
+  'LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR',
+  'A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT',
+  'OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,',
+  'SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT',
+  'LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,',
+  'DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY',
+  'THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT',
+  '(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE',
+  'OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.'
+]
+
+# Flags.
+verbose = False
+
+# Name of the dart directory when unpacking the tarball.
+versiondir = ''
+
+# Ignore Git/SVN files, checked-in binaries, backup files, etc..
+ignoredPaths = ['out', 'tools/testing/bin'
+                'third_party/7zip', 'third_party/android_tools',
+                'third_party/clang', 'third_party/d8',
+                'third_party/firefox_jsshell']
+ignoredDirs = ['.svn', '.git']
+ignoredEndings = ['.mk', '.pyc', 'Makefile', '~']
+
+def BuildOptions():
+  result = optparse.OptionParser()
+  result.add_option("-v", "--verbose",
+      help='Verbose output.',
+      default=False, action="store_true")
+  return result
+
+def Filter(tar_info):
+  _, tail = split(tar_info.name)
+  if tail in ignoredDirs:
+    return None
+  for path in ignoredPaths:
+    if tar_info.name.startswith(path):
+      return None
+  for ending in ignoredEndings:
+    if tar_info.name.endswith(ending):
+      return None
+  # Add the dart directory name with version.
+  original_name = tar_info.name
+  # Place the debian directory one level over the rest which are
+  # placed in the directory 'dart'. This enables building the Debian
+  # packages out-of-the-box.
+  tar_info.name = join(versiondir, 'dart', tar_info.name)
+  if verbose:
+    print 'Adding %s as %s' % (original_name, tar_info.name)
+  return tar_info
+
+def GenerateCopyright(filename):
+  license_lines = license
+  try:
+    # Currently the LICENSE file is part of a svn-root checkout.
+    lf = open('../LICENSE', 'r')

[kustermann, 2014/02/06 15:28:04]

'r' is the default + using context manager?

[Søren Gjesse, 2014/02/07 09:52:43]

Done.

+    license_lines = lf.read().splitlines()
+    print license_lines

[kustermann, 2014/02/06 15:28:04]

Why print?

[Søren Gjesse, 2014/02/07 09:52:43]

Debug code. Removed.

+    lf.close()
+  except:
+    pass
+
+  f = open(filename, 'w')
+  f.write('Name: dart\n')
+  f.write('Maintainer: Dart Team <misc@dartlang.org>\n')
+  f.write('Source: https://code.google.com/p/dart/\n')
+  f.write('License:\n')
+  for line in license_lines:
+    f.write(' %s\n' % line)
+  f.close()
+
+def GenerateChangeLog(filename, version):
+  f = open(filename, 'w')
+  f.write('dart (%s-1) UNRELEASED; urgency=low\n' % version)
+  f.write('\n')
+  f.write('  * Generated file.\n')
+  f.write('\n')
+  f.write(' -- Dart Team <misc@dartlang.org>  %s\n' %
+          datetime.datetime.utcnow().strftime('%a, %d %b %Y %X +0000'))

[kustermann, 2014/02/06 15:28:04]

Maybe use a multiline string? [see comment at license]

[Søren Gjesse, 2014/02/07 09:52:43]

It does not indent as well as this I think.

+  f.close()
+
+def GenerateSvnRevision(filename, svn_revision):
+  f = open(filename, 'w')

[kustermann, 2014/02/06 15:28:04]

It's usually nicer (and safer if an exception occurs, because then you don't
leak) if you use a context manager:

with open(filename, 'w') as f:
  f.write(svn_revision)

[Søren Gjesse, 2014/02/07 09:52:43]

Done.

+  f.write(svn_revision)
+  f.close()
+
+
+def CreateTarball():
+  # Generate the name of the tarfile
+  version = utils.GetVersion()
+  global versiondir
+  versiondir = 'dart-%s' % version
+  tarname = '%s.tar.gz' % versiondir
+  debian_dir = 'tools/linux_dist_support/debian'
+  # Create the tar file in the out directory.
+  tardir = utils.GetBuildDir(HOST_OS, HOST_OS)
+  if not exists(tardir):
+    makedirs(tardir)
+  tarfilename = join(tardir, tarname)

[kustermann, 2014/02/06 15:28:04]

It's slightly spooky, that you tar up the dart directory into a tarball which
lives inside the dart directory. Though you ignore "out", so it's not taring up
itself.

Since this is not invoked by gyp anyway, why do we need to put the tarball in
out/buildDir/*.tar.gz ?
You could add a commandline option for the destination of the tarball.

[Søren Gjesse, 2014/02/07 09:52:43]

I started out by generating it in the parent directory to avoid the spooky part.
However placing everything we generate in out seems best.

Adding an option for the tar file is an option, but that way one can easily
place the tar file in an included directory, and we would have to check for
that.

Changed the excluding of out to use the name from GetBuildDir.

+  print 'Creating tarball: %s' % (tarfilename)

[kustermann, 2014/02/06 15:28:04]

No parenthesis needed!

[Søren Gjesse, 2014/02/07 09:52:43]

Done.

+  with tarfile.open(tarfilename, mode='w:gz') as tar:
+    for f in listdir('.'):

[kustermann, 2014/02/06 15:28:04]

You assume this is run from the root of the dart directory I assume?

Why not ensure it by using DART_DIR as you do in other places?

[Søren Gjesse, 2014/02/07 09:52:43]

Done. The script can now be run from anywhere.

+      tar.add(f, filter=Filter)
+    for f in listdir(debian_dir):
+      tar.add(join(debian_dir, f),
+              arcname='%s/debian/%s' % (versiondir, f))
+
+    with utils.TempDir() as temp_dir:
+      # Generate and add debian/copyright
+      copyright = join(temp_dir, 'copyright')
+      GenerateCopyright(copyright)
+      tar.add(copyright, arcname='%s/debian/copyright' % versiondir)
+
+      # Generate and add debian/changelog
+      change_log = join(temp_dir, 'changelog')
+      GenerateChangeLog(change_log, version)
+      tar.add(change_log, arcname='%s/debian/changelog' % versiondir)
+
+      # For bleeding_edge add the SVN_REVISION file.
+      if utils.GetChannel() == 'be':
+        svn_revision = join(temp_dir, 'SVN_REVISION')
+        GenerateSvnRevision(svn_revision, utils.GetSVNRevision())
+        tar.add(svn_revision, arcname='%s/dart/tools/SVN_REVISION' % versiondir)
+
+def Main():
+  if HOST_OS != 'linux':
+    print 'Tarball can only be created on linux'
+    return -1
+
+  # Parse the options.
+  parser = BuildOptions()
+  (options, args) = parser.parse_args()
+  if options.verbose:
+    global verbose
+    verbose = True
+
+  CreateTarball()
+
+if __name__ == '__main__':
+  sys.exit(Main())