Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(351)

Side by Side Diff: scripts/slave/recipe_modules/chromium/resources/export_tarball.py

Issue 1981553002: Update export_tarball.py (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """ 6 """
7 This tool creates a tarball with all the sources, but without .svn directories. 7 This tool creates a tarball with all the sources, but without .git directories.
8 8
9 It can also remove files which are not strictly required for build, so that 9 It can also remove files which are not strictly required for build, so that
10 the resulting tarball can be reasonably small (last time it was ~110 MB). 10 the resulting tarball can be reasonably small (last time it was ~110 MB).
11 11
12 Example usage: 12 Example usage:
13 13
14 export_tarball.py /foo/bar 14 export_tarball.py /foo/bar
15 15
16 The above will create file /foo/bar.tar.bz2. 16 The above will create file /foo/bar.tar.bz2.
17 """ 17 """
18 18
19 import optparse 19 import optparse
20 import os 20 import os
21 import subprocess 21 import subprocess
22 import sys 22 import sys
23 import tarfile 23 import tarfile
24 24
25 25
26 NONESSENTIAL_DIRS = ( 26 NONESSENTIAL_DIRS = (
27 'breakpad/src/processor/testdata',
28 'chrome/browser/resources/tracing/tests',
29 'chrome/common/extensions/docs', 27 'chrome/common/extensions/docs',
30 'courgette/testdata',
31 'data',
32 'native_client/src/trusted/service_runtime/testdata',
33 'o3d/documentation',
34 'o3d/samples',
35 'o3d/tests',
36 'third_party/angle/samples/gles2_book',
37 'third_party/findbugs', 28 'third_party/findbugs',
38 'third_party/hunspell_dictionaries', 29 'third_party/hunspell_dictionaries',
39 'third_party/hunspell/tests', 30 'third_party/hunspell/tests',
40 'third_party/lighttpd',
41 'third_party/sqlite/src/test', 31 'third_party/sqlite/src/test',
42 'third_party/sqlite/test',
43 'third_party/vc_80',
44 'third_party/xdg-utils/tests', 32 'third_party/xdg-utils/tests',
45 'third_party/yasm/source/patched-yasm/modules/arch/x86/tests', 33 'third_party/yasm/source/patched-yasm/modules/arch/x86/tests',
46 'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests', 34 'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests',
47 'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests', 35 'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests',
48 'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests', 36 'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests',
49 'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests', 37 'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests',
50 'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests', 38 'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests',
51 'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests', 39 'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests',
52 'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests', 40 'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests',
53 'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests', 41 'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests',
54 'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests', 42 'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests',
55 'third_party/WebKit/LayoutTests', 43 'third_party/WebKit/LayoutTests',
56 'third_party/WebKit/Source/JavaScriptCore/tests',
57 'third_party/WebKit/Source/WebCore/ChangeLog',
58 'third_party/WebKit/Source/WebKit2',
59 'third_party/WebKit/Tools/Scripts', 44 'third_party/WebKit/Tools/Scripts',
60 'tools/gyp/test', 45 'tools/gyp/test',
61 'v8/test', 46 'v8/test',
62 'webkit/data/layout_tests',
63 'webkit/tools/test/reference_build',
64 ) 47 )
65 48
66 TESTDIRS = ( 49 ESSENTIAL_FILES = (
50 'chrome/test/data/webui/i18n_process_css_test.html',
51 )
52
53 TEST_DIRS = (
54 'breakpad/src/processor/testdata',
55 'chrome/browser/resources/tracing/tests',
67 'chrome/test/data', 56 'chrome/test/data',
68 'content/test/data', 57 'content/test/data',
58 'courgette/testdata',
69 'media/test/data', 59 'media/test/data',
60 'native_client/src/trusted/service_runtime/testdata',
70 'net/data', 61 'net/data',
71 ) 62 )
72 63
73 64
74 # Workaround lack of the exclude parameter in add method in python-2.4. 65 # Workaround lack of the exclude parameter in add method in python-2.4.
75 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot. 66 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot.
76 class MyTarFile(tarfile.TarFile): 67 class MyTarFile(tarfile.TarFile):
77 def set_remove_nonessential_files(self, remove): 68 def set_remove_nonessential_files(self, remove):
78 self.__remove_nonessential_files = remove 69 self.__remove_nonessential_files = remove
79 70
80 def set_verbose(self, verbose): 71 def set_verbose(self, verbose):
81 self.__verbose = verbose 72 self.__verbose = verbose
82 73
83 def set_src_dir(self, src_dir): 74 def set_src_dir(self, src_dir):
84 self.__src_dir = src_dir 75 self.__src_dir = src_dir
85 76
86 def __report_skipped(self, name): 77 def __report_skipped(self, name):
87 if self.__verbose: 78 if self.__verbose:
88 print 'D\t%s' % name 79 print 'D\t%s' % name
89 80
90 def __report_added(self, name): 81 def __report_added(self, name):
91 if self.__verbose: 82 if self.__verbose:
92 print 'A\t%s' % name 83 print 'A\t%s' % name
93 84
94 def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): 85 def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
95 head, tail = os.path.split(name) 86 _, file_name = os.path.split(name)
96 if tail in ('.svn', '.git'): 87 if file_name in ('.git', 'out'):
97 self.__report_skipped(name) 88 self.__report_skipped(name)
98 return 89 return
99 90
100 if self.__remove_nonessential_files: 91 if self.__remove_nonessential_files:
101 # WebKit change logs take quite a lot of space. This saves ~10 MB 92 # WebKit change logs take quite a lot of space. This saves ~10 MB
102 # in a bzip2-compressed tarball. 93 # in a bzip2-compressed tarball.
103 if 'ChangeLog' in name: 94 if 'ChangeLog' in name:
104 self.__report_skipped(name) 95 self.__report_skipped(name)
105 return 96 return
106 97
107 # Remove contents of non-essential directories, but preserve gyp files, 98 # Preserve GYP/GN files, and other potentially critical files, so that
108 # so that build/gyp_chromium can work. 99 # build/gyp_chromium / gn gen can work.
109 for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS): 100 rel_name = os.path.relpath(name, GetSourceDirectory())
110 dir_path = os.path.join(self.__src_dir, nonessential_dir) 101 keep_file = ('.gyp' in file_name or
111 if (name.startswith(dir_path) and 102 '.gn' in file_name or
112 os.path.isfile(name) and 103 '.isolate' in file_name or
113 'gyp' not in name and 104 '.grd' in file_name or
114 'gn' not in name and 105 rel_name in ESSENTIAL_FILES)
115 'isolate' not in name and 106
116 'grd' not in name): 107 # Remove contents of non-essential directories.
117 self.__report_skipped(name) 108 if not keep_file:
109 for nonessential_dir in (NONESSENTIAL_DIRS + TEST_DIRS):
110 if rel_name.startswith(nonessential_dir) and os.path.isfile(name):
111 self.__report_skipped(name)
118 return 112 return
119 113
120 self.__report_added(name) 114 self.__report_added(name)
121 tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive) 115 tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive)
122 116
123 117
124 def main(argv): 118 def main(argv):
125 parser = optparse.OptionParser() 119 parser = optparse.OptionParser()
126 parser.add_option("--basename") 120 parser.add_option("--basename")
127 parser.add_option("--remove-nonessential-files", 121 parser.add_option("--remove-nonessential-files",
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
159 153
160 output_fullname = args[0] + '.tar' 154 output_fullname = args[0] + '.tar'
161 output_basename = options.basename or os.path.basename(args[0]) 155 output_basename = options.basename or os.path.basename(args[0])
162 156
163 archive = MyTarFile.open(output_fullname, 'w') 157 archive = MyTarFile.open(output_fullname, 'w')
164 archive.set_remove_nonessential_files(options.remove_nonessential_files) 158 archive.set_remove_nonessential_files(options.remove_nonessential_files)
165 archive.set_verbose(options.verbose) 159 archive.set_verbose(options.verbose)
166 archive.set_src_dir(options.src_dir) 160 archive.set_src_dir(options.src_dir)
167 try: 161 try:
168 if options.test_data: 162 if options.test_data:
169 for directory in TESTDIRS: 163 for directory in TEST_DIRS:
170 archive.add(os.path.join(options.src_dir, directory), 164 archive.add(os.path.join(options.src_dir, directory),
171 arcname=os.path.join(output_basename, directory)) 165 arcname=os.path.join(output_basename, directory))
172 else: 166 else:
173 archive.add(options.src_dir, arcname=output_basename) 167 archive.add(options.src_dir, arcname=output_basename)
174 finally: 168 finally:
175 archive.close() 169 archive.close()
176 170
177 if options.progress: 171 if options.progress:
178 sys.stdout.flush() 172 sys.stdout.flush()
179 pv = subprocess.Popen( 173 pv = subprocess.Popen(
180 ['pv', '--force', output_fullname], 174 ['pv', '--force', output_fullname],
181 stdout=subprocess.PIPE, 175 stdout=subprocess.PIPE,
182 stderr=sys.stdout) 176 stderr=sys.stdout)
183 with open(output_fullname + '.xz', 'w') as f: 177 with open(output_fullname + '.xz', 'w') as f:
184 rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f) 178 rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f)
185 pv.wait() 179 pv.wait()
186 else: 180 else:
187 rc = subprocess.call(['xz', '-9', output_fullname]) 181 rc = subprocess.call(['xz', '-9', output_fullname])
188 182
189 if rc != 0: 183 if rc != 0:
190 print 'xz -9 failed!' 184 print 'xz -9 failed!'
191 return 1 185 return 1
192 186
193 return 0 187 return 0
194 188
195 189
196 if __name__ == "__main__": 190 if __name__ == "__main__":
197 sys.exit(main(sys.argv[1:])) 191 sys.exit(main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698