OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright 2016 The Chromium Authors. All rights reserved. | 2 # Copyright 2016 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """ | 6 """ |
7 Convert the ASCII download_file_types.asciipb proto into a binary resource. | 7 Convert the ASCII download_file_types.asciipb proto into a binary resource. |
8 | 8 |
9 We generate a separate variant of the binary proto for each platform, | 9 We generate a separate variant of the binary proto for each platform, |
10 each which contains only the values that platform needs. | 10 each which contains only the values that platform needs. |
11 """ | 11 """ |
12 | 12 |
13 import optparse | 13 import optparse |
14 import os | |
14 import re | 15 import re |
15 import subprocess | 16 import subprocess |
16 import sys | 17 import sys |
18 import traceback | |
17 | 19 |
18 | 20 |
19 def ImportProtoModules(paths): | 21 def ImportProtoModules(paths): |
20 """Import the protobuf modiles we need. |paths| is list of import paths""" | 22 """Import the protobuf modiles we need. |paths| is list of import paths""" |
21 for path in paths: | 23 for path in paths: |
22 # Put the path to our proto libraries in front, so that we don't use system | 24 # Put the path to our proto libraries in front, so that we don't use system |
23 # protobuf. | 25 # protobuf. |
24 sys.path.insert(1, path) | 26 sys.path.insert(1, path) |
25 | 27 |
26 import download_file_types_pb2 as config_pb2 | 28 import download_file_types_pb2 as config_pb2 |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
86 | 88 |
87 # Now clear out the full list and replace it with 1 entry. | 89 # Now clear out the full list and replace it with 1 entry. |
88 del file_type.platform_settings[:] | 90 del file_type.platform_settings[:] |
89 new_setting = file_type.platform_settings.add() | 91 new_setting = file_type.platform_settings.add() |
90 new_setting.CopyFrom(setting_match) | 92 new_setting.CopyFrom(setting_match) |
91 new_setting.ClearField('platform') | 93 new_setting.ClearField('platform') |
92 | 94 |
93 | 95 |
94 def FilterPbForPlatform(full_pb, platform_type): | 96 def FilterPbForPlatform(full_pb, platform_type): |
95 """ Return a filtered protobuf for this platform_type """ | 97 """ Return a filtered protobuf for this platform_type """ |
98 assert type(platform_type) is int, "Bad platform_type type" | |
96 | 99 |
97 new_pb = config_pb2.DownloadFileTypeConfig(); | 100 new_pb = config_pb2.DownloadFileTypeConfig(); |
98 new_pb.CopyFrom(full_pb) | 101 new_pb.CopyFrom(full_pb) |
99 | 102 |
100 # Ensure there's only one platform_settings for the default. | 103 # Ensure there's only one platform_settings for the default. |
101 PrunePlatformSettings(new_pb.default_file_type, None, platform_type) | 104 PrunePlatformSettings(new_pb.default_file_type, None, platform_type) |
102 | 105 |
103 # This can be extended if we want to match weird extensions. | 106 # This can be extended if we want to match weird extensions. |
104 # Just no dots, non-UTF8, or uppercase chars. | 107 # Just no dots, non-UTF8, or uppercase chars. |
105 invalid_char_re = re.compile('[^a-z0-9_-]') | 108 invalid_char_re = re.compile('[^a-z0-9_-]') |
(...skipping 14 matching lines...) Expand all Loading... | |
120 file_type.extension, file_type.uma_value)) | 123 file_type.extension, file_type.uma_value)) |
121 uma_values_used.add(file_type.uma_value) | 124 uma_values_used.add(file_type.uma_value) |
122 | 125 |
123 # Modify file_type to include only the best match platform_setting. | 126 # Modify file_type to include only the best match platform_setting. |
124 PrunePlatformSettings( | 127 PrunePlatformSettings( |
125 file_type, new_pb.default_file_type.platform_settings[0], platform_type) | 128 file_type, new_pb.default_file_type.platform_settings[0], platform_type) |
126 | 129 |
127 return new_pb | 130 return new_pb |
128 | 131 |
129 | 132 |
130 def GeneratBinaryProtos(opts): | 133 def FilterForPlatformAndWrite(full_pb, platform_type, outfile): |
134 """ Filter and write out a file for this platform """ | |
135 # Filter it | |
136 filtered_pb = FilterPbForPlatform(full_pb, platform_type); | |
137 | |
138 # Serialize it | |
139 binary_pb_str = filtered_pb.SerializeToString() | |
140 | |
141 # Write it to disk | |
142 open(outfile, 'wb').write(binary_pb_str) | |
143 | |
144 | |
145 def MakeSubDirs(outfile): | |
146 """ Make the subdirectories needed to create file |outfile| """ | |
147 dirname = os.path.dirname(outfile) | |
148 if not os.path.exists(dirname): | |
149 os.makedirs(dirname) | |
150 | |
151 | |
152 def GenerateBinaryProtos(opts): | |
153 """ Read the ASCII proto and generate one or more binary protos. """ | |
131 # Read the ASCII | 154 # Read the ASCII |
132 ifile = open(opts.infile, 'r') | 155 ifile = open(opts.infile, 'r') |
133 ascii_pb_str = ifile.read() | 156 ascii_pb_str = ifile.read() |
134 ifile.close() | 157 ifile.close() |
135 | 158 |
136 # Parse it into a structure PB | 159 # Parse it into a structured PB |
137 pb = config_pb2.DownloadFileTypeConfig() | 160 full_pb = config_pb2.DownloadFileTypeConfig() |
138 text_format.Merge(ascii_pb_str, pb) | 161 text_format.Merge(ascii_pb_str, full_pb) |
139 | 162 |
140 ValidatePb(pb); | 163 ValidatePb(full_pb); |
141 platform_type = PlatformTypes()[opts.type] | |
142 filtered_pb = FilterPbForPlatform(pb, platform_type); | |
143 | 164 |
144 # Serialize it | 165 if opts.type is not None: |
145 binary_pb_str = filtered_pb.SerializeToString() | 166 # Just one platform type |
167 platform_enum = PlatformTypes()[opts.type] | |
168 outfile = os.path.join(opts.outdir, opts.outbasename) | |
169 FilterForPlatformAndWrite(full_pb, platform_enum, outfile) | |
170 else: | |
171 # Make a separate file for each platform | |
172 for platform_type, platform_enum in PlatformTypes().iteritems(): | |
173 # e.g. .../all/77/chromeos/download_file_types.pb | |
174 outfile = os.path.join(opts.outdir, | |
175 str(full_pb.version_id), | |
176 platform_type, | |
177 opts.outbasename) | |
178 MakeSubDirs(outfile) | |
179 FilterForPlatformAndWrite(full_pb, platform_enum, outfile) | |
146 | 180 |
147 # Write it to disk | 181 print "\n\nTo push these files, run the following:" |
brucedawson
2016/06/02 00:45:07
This message appears to print as part of a normal
| |
148 open(opts.outfile, 'wb').write(binary_pb_str) | 182 print ("python " + |
183 "chrome/browser/resources/safe_browsing/push_file_type_proto.py " + | |
184 "-d " + os.path.abspath(opts.outdir)) | |
185 print "\n\n" | |
149 | 186 |
150 | 187 |
151 def main(): | 188 def main(): |
152 parser = optparse.OptionParser() | 189 parser = optparse.OptionParser() |
153 # TODO(nparker): Remove this once the bug is fixed. | 190 # TODO(nparker): Remove this once the bug is fixed. |
154 parser.add_option('-w', '--wrap', action="store_true", default=False, | 191 parser.add_option('-w', '--wrap', action="store_true", default=False, |
155 help='Wrap this script in another python ' | 192 help='Wrap this script in another python ' |
156 'execution to disable site-packages. This is a ' | 193 'execution to disable site-packages. This is a ' |
157 'fix for http://crbug.com/605592') | 194 'fix for http://crbug.com/605592') |
195 | |
196 parser.add_option('-a', '--all', action="store_true", default=False, | |
197 help='Write a separate file for every platform. ' | |
198 'Outfile must have a %d for version and %s for platform.') | |
158 parser.add_option('-t', '--type', | 199 parser.add_option('-t', '--type', |
159 help='The platform type. One of android, chromeos, ' + | 200 help='The platform type. One of android, chromeos, ' + |
160 'linux, mac, win') | 201 'linux, mac, win') |
161 parser.add_option('-i', '--infile', | 202 parser.add_option('-i', '--infile', |
162 help='The ASCII DownloadFileType-proto file to read.') | 203 help='The ASCII DownloadFileType-proto file to read.') |
163 parser.add_option('-o', '--outfile', | 204 parser.add_option('-d', '--outdir', |
164 help='The binary file to write to.') | 205 help='Directory underwhich binary file(s) will be written') |
206 parser.add_option('-o', '--outbasename', | |
207 help='Basename of the binary file to write to.') | |
165 parser.add_option('-p', '--path', action="append", | 208 parser.add_option('-p', '--path', action="append", |
166 help='Repeat this as needed. Directory(s) containing ' + | 209 help='Repeat this as needed. Directory(s) containing ' + |
167 'the download_file_types_pb2.py and ' + | 210 'the download_file_types_pb2.py and ' + |
168 'google.protobuf.text_format modules') | 211 'google.protobuf.text_format modules') |
169 (opts, args) = parser.parse_args() | 212 (opts, args) = parser.parse_args() |
170 if opts.infile is None or opts.outfile is None: | 213 if opts.infile is None or opts.outdir is None or opts.outbasename is None: |
171 parser.print_help() | 214 parser.print_help() |
172 return 1 | 215 return 1 |
173 | 216 |
174 if opts.wrap: | 217 if opts.wrap: |
175 # Run this script again with different args to the interpreter. | 218 # Run this script again with different args to the interpreter. |
176 command = [sys.executable, '-S', '-s', sys.argv[0]] | 219 command = [sys.executable, '-S', '-s', sys.argv[0]] |
177 command += ['-t', opts.type] | 220 if opts.type is not None: |
221 command += ['-t', opts.type] | |
222 if opts.all: | |
223 command += ['-a'] | |
178 command += ['-i', opts.infile] | 224 command += ['-i', opts.infile] |
179 command += ['-o', opts.outfile] | 225 command += ['-d', opts.outdir] |
226 command += ['-o', opts.outbasename] | |
180 for path in opts.path: | 227 for path in opts.path: |
181 command += ['-p', path] | 228 command += ['-p', path] |
182 sys.exit(subprocess.call(command)) | 229 sys.exit(subprocess.call(command)) |
183 | 230 |
184 ImportProtoModules(opts.path) | 231 ImportProtoModules(opts.path) |
185 | 232 |
186 if (opts.type not in PlatformTypes()): | 233 if (not opts.all and opts.type not in PlatformTypes()): |
187 print "ERROR: Unknown platform type '%s'" % opts.type | 234 print "ERROR: Unknown platform type '%s'" % opts.type |
188 parser.print_help() | 235 parser.print_help() |
189 return 1 | 236 return 1 |
190 | 237 |
238 if (bool(opts.all) == bool(opts.type)): | |
239 print "ERROR: Need exactly one of --type or --all" | |
240 parser.print_help() | |
241 return 1 | |
242 | |
191 try: | 243 try: |
192 GeneratBinaryProtos(opts) | 244 GenerateBinaryProtos(opts) |
193 except Exception as e: | 245 except Exception as e: |
194 print "ERROR: Failed to render binary version of %s:\n %s\n" % ( | 246 print "ERROR: Failed to render binary version of %s:\n %s\n%s" % ( |
195 opts.infile, str(e)) | 247 opts.infile, str(e), traceback.format_exc()) |
196 return 1 | 248 return 1 |
197 | 249 |
198 | 250 |
199 if __name__ == '__main__': | 251 if __name__ == '__main__': |
200 sys.exit(main()) | 252 sys.exit(main()) |
OLD | NEW |