| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """ | 6 """ |
| 7 This tool creates a tarball with all the sources, but without .git directories. | 7 This tool creates a tarball with all the sources, but without .svn directories. |
| 8 | 8 |
| 9 It can also remove files which are not strictly required for build, so that | 9 It can also remove files which are not strictly required for build, so that |
| 10 the resulting tarball can be reasonably small (last time it was ~110 MB). | 10 the resulting tarball can be reasonably small (last time it was ~110 MB). |
| 11 | 11 |
| 12 Example usage: | 12 Example usage: |
| 13 | 13 |
| 14 export_tarball.py /foo/bar | 14 export_tarball.py /foo/bar |
| 15 | 15 |
| 16 The above will create file /foo/bar.tar.bz2. | 16 The above will create file /foo/bar.tar.bz2. |
| 17 """ | 17 """ |
| 18 | 18 |
| 19 import optparse | 19 import optparse |
| 20 import os | 20 import os |
| 21 import subprocess | 21 import subprocess |
| 22 import sys | 22 import sys |
| 23 import tarfile | 23 import tarfile |
| 24 | 24 |
| 25 | 25 |
| 26 NONESSENTIAL_DIRS = ( | 26 NONESSENTIAL_DIRS = ( |
| 27 'breakpad/src/processor/testdata', |
| 28 'chrome/browser/resources/tracing/tests', |
| 27 'chrome/common/extensions/docs', | 29 'chrome/common/extensions/docs', |
| 30 'courgette/testdata', |
| 31 'data', |
| 32 'native_client/src/trusted/service_runtime/testdata', |
| 33 'o3d/documentation', |
| 34 'o3d/samples', |
| 35 'o3d/tests', |
| 36 'third_party/angle/samples/gles2_book', |
| 28 'third_party/findbugs', | 37 'third_party/findbugs', |
| 29 'third_party/hunspell_dictionaries', | 38 'third_party/hunspell_dictionaries', |
| 30 'third_party/hunspell/tests', | 39 'third_party/hunspell/tests', |
| 40 'third_party/lighttpd', |
| 31 'third_party/sqlite/src/test', | 41 'third_party/sqlite/src/test', |
| 42 'third_party/sqlite/test', |
| 43 'third_party/vc_80', |
| 32 'third_party/xdg-utils/tests', | 44 'third_party/xdg-utils/tests', |
| 33 'third_party/yasm/source/patched-yasm/modules/arch/x86/tests', | 45 'third_party/yasm/source/patched-yasm/modules/arch/x86/tests', |
| 34 'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests', | 46 'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests', |
| 35 'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests', | 47 'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests', |
| 36 'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests', | 48 'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests', |
| 37 'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests', | 49 'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests', |
| 38 'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests', | 50 'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests', |
| 39 'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests', | 51 'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests', |
| 40 'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests', | 52 'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests', |
| 41 'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests', | 53 'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests', |
| 42 'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests', | 54 'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests', |
| 43 'third_party/WebKit/LayoutTests', | 55 'third_party/WebKit/LayoutTests', |
| 56 'third_party/WebKit/Source/JavaScriptCore/tests', |
| 57 'third_party/WebKit/Source/WebCore/ChangeLog', |
| 58 'third_party/WebKit/Source/WebKit2', |
| 44 'third_party/WebKit/Tools/Scripts', | 59 'third_party/WebKit/Tools/Scripts', |
| 45 'tools/gyp/test', | 60 'tools/gyp/test', |
| 46 'v8/test', | 61 'v8/test', |
| 62 'webkit/data/layout_tests', |
| 63 'webkit/tools/test/reference_build', |
| 47 ) | 64 ) |
| 48 | 65 |
| 49 ESSENTIAL_FILES = ( | 66 TESTDIRS = ( |
| 50 'chrome/test/data/webui/i18n_process_css_test.html', | |
| 51 ) | |
| 52 | |
| 53 TEST_DIRS = ( | |
| 54 'breakpad/src/processor/testdata', | |
| 55 'chrome/browser/resources/tracing/tests', | |
| 56 'chrome/test/data', | 67 'chrome/test/data', |
| 57 'content/test/data', | 68 'content/test/data', |
| 58 'courgette/testdata', | |
| 59 'media/test/data', | 69 'media/test/data', |
| 60 'native_client/src/trusted/service_runtime/testdata', | |
| 61 'net/data', | 70 'net/data', |
| 62 ) | 71 ) |
| 63 | 72 |
| 64 | 73 |
| 65 # Workaround lack of the exclude parameter in add method in python-2.4. | 74 # Workaround lack of the exclude parameter in add method in python-2.4. |
| 66 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot. | 75 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot. |
| 67 class MyTarFile(tarfile.TarFile): | 76 class MyTarFile(tarfile.TarFile): |
| 68 def set_remove_nonessential_files(self, remove): | 77 def set_remove_nonessential_files(self, remove): |
| 69 self.__remove_nonessential_files = remove | 78 self.__remove_nonessential_files = remove |
| 70 | 79 |
| 71 def set_verbose(self, verbose): | 80 def set_verbose(self, verbose): |
| 72 self.__verbose = verbose | 81 self.__verbose = verbose |
| 73 | 82 |
| 74 def set_src_dir(self, src_dir): | 83 def set_src_dir(self, src_dir): |
| 75 self.__src_dir = src_dir | 84 self.__src_dir = src_dir |
| 76 | 85 |
| 77 def __report_skipped(self, name): | 86 def __report_skipped(self, name): |
| 78 if self.__verbose: | 87 if self.__verbose: |
| 79 print 'D\t%s' % name | 88 print 'D\t%s' % name |
| 80 | 89 |
| 81 def __report_added(self, name): | 90 def __report_added(self, name): |
| 82 if self.__verbose: | 91 if self.__verbose: |
| 83 print 'A\t%s' % name | 92 print 'A\t%s' % name |
| 84 | 93 |
| 85 def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): | 94 def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): |
| 86 _, file_name = os.path.split(name) | 95 head, tail = os.path.split(name) |
| 87 if file_name in ('.git', 'out'): | 96 if tail in ('.svn', '.git'): |
| 88 self.__report_skipped(name) | 97 self.__report_skipped(name) |
| 89 return | 98 return |
| 90 | 99 |
| 91 if self.__remove_nonessential_files: | 100 if self.__remove_nonessential_files: |
| 92 # WebKit change logs take quite a lot of space. This saves ~10 MB | 101 # WebKit change logs take quite a lot of space. This saves ~10 MB |
| 93 # in a bzip2-compressed tarball. | 102 # in a bzip2-compressed tarball. |
| 94 if 'ChangeLog' in name: | 103 if 'ChangeLog' in name: |
| 95 self.__report_skipped(name) | 104 self.__report_skipped(name) |
| 96 return | 105 return |
| 97 | 106 |
| 98 # Preserve GYP/GN files, and other potentially critical files, so that | 107 # Remove contents of non-essential directories, but preserve gyp files, |
| 99 # build/gyp_chromium / gn gen can work. | 108 # so that build/gyp_chromium can work. |
| 100 rel_name = os.path.relpath(name, GetSourceDirectory()) | 109 for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS): |
| 101 keep_file = ('.gyp' in file_name or | 110 dir_path = os.path.join(self.__src_dir, nonessential_dir) |
| 102 '.gn' in file_name or | 111 if (name.startswith(dir_path) and |
| 103 '.isolate' in file_name or | 112 os.path.isfile(name) and |
| 104 '.grd' in file_name or | 113 'gyp' not in name and |
| 105 rel_name in ESSENTIAL_FILES) | 114 'gn' not in name and |
| 106 | 115 'isolate' not in name and |
| 107 # Remove contents of non-essential directories. | 116 'grd' not in name): |
| 108 if not keep_file: | 117 self.__report_skipped(name) |
| 109 for nonessential_dir in (NONESSENTIAL_DIRS + TEST_DIRS): | |
| 110 if rel_name.startswith(nonessential_dir) and os.path.isfile(name): | |
| 111 self.__report_skipped(name) | |
| 112 return | 118 return |
| 113 | 119 |
| 114 self.__report_added(name) | 120 self.__report_added(name) |
| 115 tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive) | 121 tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive) |
| 116 | 122 |
| 117 | 123 |
| 118 def main(argv): | 124 def main(argv): |
| 119 parser = optparse.OptionParser() | 125 parser = optparse.OptionParser() |
| 120 parser.add_option("--basename") | 126 parser.add_option("--basename") |
| 121 parser.add_option("--remove-nonessential-files", | 127 parser.add_option("--remove-nonessential-files", |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 153 | 159 |
| 154 output_fullname = args[0] + '.tar' | 160 output_fullname = args[0] + '.tar' |
| 155 output_basename = options.basename or os.path.basename(args[0]) | 161 output_basename = options.basename or os.path.basename(args[0]) |
| 156 | 162 |
| 157 archive = MyTarFile.open(output_fullname, 'w') | 163 archive = MyTarFile.open(output_fullname, 'w') |
| 158 archive.set_remove_nonessential_files(options.remove_nonessential_files) | 164 archive.set_remove_nonessential_files(options.remove_nonessential_files) |
| 159 archive.set_verbose(options.verbose) | 165 archive.set_verbose(options.verbose) |
| 160 archive.set_src_dir(options.src_dir) | 166 archive.set_src_dir(options.src_dir) |
| 161 try: | 167 try: |
| 162 if options.test_data: | 168 if options.test_data: |
| 163 for directory in TEST_DIRS: | 169 for directory in TESTDIRS: |
| 164 archive.add(os.path.join(options.src_dir, directory), | 170 archive.add(os.path.join(options.src_dir, directory), |
| 165 arcname=os.path.join(output_basename, directory)) | 171 arcname=os.path.join(output_basename, directory)) |
| 166 else: | 172 else: |
| 167 archive.add(options.src_dir, arcname=output_basename) | 173 archive.add(options.src_dir, arcname=output_basename) |
| 168 finally: | 174 finally: |
| 169 archive.close() | 175 archive.close() |
| 170 | 176 |
| 171 if options.progress: | 177 if options.progress: |
| 172 sys.stdout.flush() | 178 sys.stdout.flush() |
| 173 pv = subprocess.Popen( | 179 pv = subprocess.Popen( |
| 174 ['pv', '--force', output_fullname], | 180 ['pv', '--force', output_fullname], |
| 175 stdout=subprocess.PIPE, | 181 stdout=subprocess.PIPE, |
| 176 stderr=sys.stdout) | 182 stderr=sys.stdout) |
| 177 with open(output_fullname + '.xz', 'w') as f: | 183 with open(output_fullname + '.xz', 'w') as f: |
| 178 rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f) | 184 rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f) |
| 179 pv.wait() | 185 pv.wait() |
| 180 else: | 186 else: |
| 181 rc = subprocess.call(['xz', '-9', output_fullname]) | 187 rc = subprocess.call(['xz', '-9', output_fullname]) |
| 182 | 188 |
| 183 if rc != 0: | 189 if rc != 0: |
| 184 print 'xz -9 failed!' | 190 print 'xz -9 failed!' |
| 185 return 1 | 191 return 1 |
| 186 | 192 |
| 187 return 0 | 193 return 0 |
| 188 | 194 |
| 189 | 195 |
| 190 if __name__ == "__main__": | 196 if __name__ == "__main__": |
| 191 sys.exit(main(sys.argv[1:])) | 197 sys.exit(main(sys.argv[1:])) |
| OLD | NEW |