Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(4)

Side by Side Diff: scripts/slave/recipe_modules/chromium/resources/export_tarball.py

Issue 1984053003: Revert of Update export_tarball.py (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """ 6 """
7 This tool creates a tarball with all the sources, but without .git directories. 7 This tool creates a tarball with all the sources, but without .svn directories.
8 8
9 It can also remove files which are not strictly required for build, so that 9 It can also remove files which are not strictly required for build, so that
10 the resulting tarball can be reasonably small (last time it was ~110 MB). 10 the resulting tarball can be reasonably small (last time it was ~110 MB).
11 11
12 Example usage: 12 Example usage:
13 13
14 export_tarball.py /foo/bar 14 export_tarball.py /foo/bar
15 15
16 The above will create file /foo/bar.tar.bz2. 16 The above will create file /foo/bar.tar.bz2.
17 """ 17 """
18 18
19 import optparse 19 import optparse
20 import os 20 import os
21 import subprocess 21 import subprocess
22 import sys 22 import sys
23 import tarfile 23 import tarfile
24 24
25 25
26 NONESSENTIAL_DIRS = ( 26 NONESSENTIAL_DIRS = (
27 'breakpad/src/processor/testdata',
28 'chrome/browser/resources/tracing/tests',
27 'chrome/common/extensions/docs', 29 'chrome/common/extensions/docs',
30 'courgette/testdata',
31 'data',
32 'native_client/src/trusted/service_runtime/testdata',
33 'o3d/documentation',
34 'o3d/samples',
35 'o3d/tests',
36 'third_party/angle/samples/gles2_book',
28 'third_party/findbugs', 37 'third_party/findbugs',
29 'third_party/hunspell_dictionaries', 38 'third_party/hunspell_dictionaries',
30 'third_party/hunspell/tests', 39 'third_party/hunspell/tests',
40 'third_party/lighttpd',
31 'third_party/sqlite/src/test', 41 'third_party/sqlite/src/test',
42 'third_party/sqlite/test',
43 'third_party/vc_80',
32 'third_party/xdg-utils/tests', 44 'third_party/xdg-utils/tests',
33 'third_party/yasm/source/patched-yasm/modules/arch/x86/tests', 45 'third_party/yasm/source/patched-yasm/modules/arch/x86/tests',
34 'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests', 46 'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests',
35 'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests', 47 'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests',
36 'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests', 48 'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests',
37 'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests', 49 'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests',
38 'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests', 50 'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests',
39 'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests', 51 'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests',
40 'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests', 52 'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests',
41 'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests', 53 'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests',
42 'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests', 54 'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests',
43 'third_party/WebKit/LayoutTests', 55 'third_party/WebKit/LayoutTests',
56 'third_party/WebKit/Source/JavaScriptCore/tests',
57 'third_party/WebKit/Source/WebCore/ChangeLog',
58 'third_party/WebKit/Source/WebKit2',
44 'third_party/WebKit/Tools/Scripts', 59 'third_party/WebKit/Tools/Scripts',
45 'tools/gyp/test', 60 'tools/gyp/test',
46 'v8/test', 61 'v8/test',
62 'webkit/data/layout_tests',
63 'webkit/tools/test/reference_build',
47 ) 64 )
48 65
49 ESSENTIAL_FILES = ( 66 TESTDIRS = (
50 'chrome/test/data/webui/i18n_process_css_test.html',
51 )
52
53 TEST_DIRS = (
54 'breakpad/src/processor/testdata',
55 'chrome/browser/resources/tracing/tests',
56 'chrome/test/data', 67 'chrome/test/data',
57 'content/test/data', 68 'content/test/data',
58 'courgette/testdata',
59 'media/test/data', 69 'media/test/data',
60 'native_client/src/trusted/service_runtime/testdata',
61 'net/data', 70 'net/data',
62 ) 71 )
63 72
64 73
65 # Workaround lack of the exclude parameter in add method in python-2.4. 74 # Workaround lack of the exclude parameter in add method in python-2.4.
66 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot. 75 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot.
67 class MyTarFile(tarfile.TarFile): 76 class MyTarFile(tarfile.TarFile):
68 def set_remove_nonessential_files(self, remove): 77 def set_remove_nonessential_files(self, remove):
69 self.__remove_nonessential_files = remove 78 self.__remove_nonessential_files = remove
70 79
71 def set_verbose(self, verbose): 80 def set_verbose(self, verbose):
72 self.__verbose = verbose 81 self.__verbose = verbose
73 82
74 def set_src_dir(self, src_dir): 83 def set_src_dir(self, src_dir):
75 self.__src_dir = src_dir 84 self.__src_dir = src_dir
76 85
77 def __report_skipped(self, name): 86 def __report_skipped(self, name):
78 if self.__verbose: 87 if self.__verbose:
79 print 'D\t%s' % name 88 print 'D\t%s' % name
80 89
81 def __report_added(self, name): 90 def __report_added(self, name):
82 if self.__verbose: 91 if self.__verbose:
83 print 'A\t%s' % name 92 print 'A\t%s' % name
84 93
85 def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): 94 def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
86 _, file_name = os.path.split(name) 95 head, tail = os.path.split(name)
87 if file_name in ('.git', 'out'): 96 if tail in ('.svn', '.git'):
88 self.__report_skipped(name) 97 self.__report_skipped(name)
89 return 98 return
90 99
91 if self.__remove_nonessential_files: 100 if self.__remove_nonessential_files:
92 # WebKit change logs take quite a lot of space. This saves ~10 MB 101 # WebKit change logs take quite a lot of space. This saves ~10 MB
93 # in a bzip2-compressed tarball. 102 # in a bzip2-compressed tarball.
94 if 'ChangeLog' in name: 103 if 'ChangeLog' in name:
95 self.__report_skipped(name) 104 self.__report_skipped(name)
96 return 105 return
97 106
98 # Preserve GYP/GN files, and other potentially critical files, so that 107 # Remove contents of non-essential directories, but preserve gyp files,
99 # build/gyp_chromium / gn gen can work. 108 # so that build/gyp_chromium can work.
100 rel_name = os.path.relpath(name, GetSourceDirectory()) 109 for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS):
101 keep_file = ('.gyp' in file_name or 110 dir_path = os.path.join(self.__src_dir, nonessential_dir)
102 '.gn' in file_name or 111 if (name.startswith(dir_path) and
103 '.isolate' in file_name or 112 os.path.isfile(name) and
104 '.grd' in file_name or 113 'gyp' not in name and
105 rel_name in ESSENTIAL_FILES) 114 'gn' not in name and
106 115 'isolate' not in name and
107 # Remove contents of non-essential directories. 116 'grd' not in name):
108 if not keep_file: 117 self.__report_skipped(name)
109 for nonessential_dir in (NONESSENTIAL_DIRS + TEST_DIRS):
110 if rel_name.startswith(nonessential_dir) and os.path.isfile(name):
111 self.__report_skipped(name)
112 return 118 return
113 119
114 self.__report_added(name) 120 self.__report_added(name)
115 tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive) 121 tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive)
116 122
117 123
118 def main(argv): 124 def main(argv):
119 parser = optparse.OptionParser() 125 parser = optparse.OptionParser()
120 parser.add_option("--basename") 126 parser.add_option("--basename")
121 parser.add_option("--remove-nonessential-files", 127 parser.add_option("--remove-nonessential-files",
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
153 159
154 output_fullname = args[0] + '.tar' 160 output_fullname = args[0] + '.tar'
155 output_basename = options.basename or os.path.basename(args[0]) 161 output_basename = options.basename or os.path.basename(args[0])
156 162
157 archive = MyTarFile.open(output_fullname, 'w') 163 archive = MyTarFile.open(output_fullname, 'w')
158 archive.set_remove_nonessential_files(options.remove_nonessential_files) 164 archive.set_remove_nonessential_files(options.remove_nonessential_files)
159 archive.set_verbose(options.verbose) 165 archive.set_verbose(options.verbose)
160 archive.set_src_dir(options.src_dir) 166 archive.set_src_dir(options.src_dir)
161 try: 167 try:
162 if options.test_data: 168 if options.test_data:
163 for directory in TEST_DIRS: 169 for directory in TESTDIRS:
164 archive.add(os.path.join(options.src_dir, directory), 170 archive.add(os.path.join(options.src_dir, directory),
165 arcname=os.path.join(output_basename, directory)) 171 arcname=os.path.join(output_basename, directory))
166 else: 172 else:
167 archive.add(options.src_dir, arcname=output_basename) 173 archive.add(options.src_dir, arcname=output_basename)
168 finally: 174 finally:
169 archive.close() 175 archive.close()
170 176
171 if options.progress: 177 if options.progress:
172 sys.stdout.flush() 178 sys.stdout.flush()
173 pv = subprocess.Popen( 179 pv = subprocess.Popen(
174 ['pv', '--force', output_fullname], 180 ['pv', '--force', output_fullname],
175 stdout=subprocess.PIPE, 181 stdout=subprocess.PIPE,
176 stderr=sys.stdout) 182 stderr=sys.stdout)
177 with open(output_fullname + '.xz', 'w') as f: 183 with open(output_fullname + '.xz', 'w') as f:
178 rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f) 184 rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f)
179 pv.wait() 185 pv.wait()
180 else: 186 else:
181 rc = subprocess.call(['xz', '-9', output_fullname]) 187 rc = subprocess.call(['xz', '-9', output_fullname])
182 188
183 if rc != 0: 189 if rc != 0:
184 print 'xz -9 failed!' 190 print 'xz -9 failed!'
185 return 1 191 return 1
186 192
187 return 0 193 return 0
188 194
189 195
190 if __name__ == "__main__": 196 if __name__ == "__main__":
191 sys.exit(main(sys.argv[1:])) 197 sys.exit(main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698