OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
3 # for details. All rights reserved. Use of this source code is governed by a | |
4 # BSD-style license that can be found in the LICENSE file. | |
5 | |
6 import copy | |
7 import database | |
8 import idlparser | |
9 import logging | |
10 import multiprocessing | |
11 import os | |
12 import os.path | |
13 import re | |
14 | |
15 from idlnode import * | |
16 | |
17 _logger = logging.getLogger('databasebuilder') | |
18 | |
19 # Used in source annotations to specify the parent interface declaring | |
20 # a displaced declaration. The 'via' attribute specifies the parent interface | |
21 # which implements a displaced declaration. | |
22 _VIA_ANNOTATION_ATTR_NAME = 'via' | |
23 | |
24 | |
25 class DatabaseBuilderOptions(object): | |
26 """Used in specifying options when importing new interfaces""" | |
27 | |
28 def __init__(self, | |
29 idl_syntax=idlparser.WEBIDL_SYNTAX, | |
30 idl_defines=[], | |
31 source=None, source_attributes={}, | |
32 rename_operation_arguments_on_merge=False, | |
33 add_new_interfaces=True, | |
34 obsolete_old_declarations=False): | |
35 """Constructor. | |
36 Args: | |
37 idl_syntax -- the syntax of the IDL file that is imported. | |
38 idl_defines -- list of definitions for the idl gcc pre-processor | |
39 source -- the origin of the IDL file, used for annotating the | |
40 database. | |
41 source_attributes -- this map of attributes is used as | |
42 annotation attributes. | |
43 rename_operation_arguments_on_merge -- if True, will rename | |
44 operation arguments when merging using the new name rather | |
45 than the old. | |
46 add_new_interfaces -- when False, if an interface is a new | |
47 addition, it will be ignored. | |
48 obsolete_old_declarations -- when True, if a declaration | |
49 from a certain source is not re-declared, it will be removed. | |
50 """ | |
51 self.source = source | |
52 self.source_attributes = source_attributes | |
53 self.idl_syntax = idl_syntax | |
54 self.idl_defines = idl_defines | |
55 self.rename_operation_arguments_on_merge = \ | |
56 rename_operation_arguments_on_merge | |
57 self.add_new_interfaces = add_new_interfaces | |
58 self.obsolete_old_declarations = obsolete_old_declarations | |
59 | |
60 | |
61 def _load_idl_file(file_name, import_options): | |
62 """Loads an IDL file into memory""" | |
63 idl_parser = idlparser.IDLParser(import_options.idl_syntax) | |
64 | |
65 try: | |
66 f = open(file_name, 'r') | |
67 content = f.read() | |
68 f.close() | |
69 | |
70 idl_ast = idl_parser.parse( | |
71 content, | |
72 defines=import_options.idl_defines) | |
73 return IDLFile(idl_ast, file_name) | |
74 except SyntaxError, e: | |
75 raise RuntimeError('Failed to load file %s: %s' | |
76 % (file_name, e)) | |
77 | |
78 | |
79 class DatabaseBuilder(object): | |
80 def __init__(self, database): | |
81 """DatabaseBuilder is used for importing and merging interfaces into | |
82 the Database""" | |
83 self._database = database | |
84 self._imported_interfaces = [] | |
85 self._impl_stmts = [] | |
86 | |
87 def _resolve_type_defs(self, idl_file): | |
88 type_def_map = {} | |
89 # build map | |
90 for type_def in idl_file.all(IDLTypeDef): | |
91 if type_def.type.id != type_def.id: # sanity check | |
92 type_def_map[type_def.id] = type_def.type.id | |
93 # use the map | |
94 for type_node in idl_file.all(IDLType): | |
95 while type_node.id in type_def_map: | |
96 type_node.id = type_def_map[type_node.id] | |
97 | |
98 def _strip_ext_attributes(self, idl_file): | |
99 """Strips unuseful extended attributes.""" | |
100 for ext_attrs in idl_file.all(IDLExtAttrs): | |
101 # TODO: Decide which attributes are uninteresting. | |
102 pass | |
103 | |
104 def _rename_types(self, idl_file, import_options): | |
105 """Rename interface and type names with names provided in the | |
106 options. Also clears scopes from scoped names""" | |
107 | |
108 strip_modules = lambda name: name.split('::')[-1] | |
109 | |
110 def rename_node(idl_node): | |
111 idl_node.reset_id(strip_modules(idl_node.id)) | |
112 | |
113 def rename_ext_attrs(ext_attrs_node): | |
114 for type_valued_attribute_name in ['Supplemental']: | |
115 if type_valued_attribute_name in ext_attrs_node: | |
116 value = ext_attrs_node[type_valued_attribute_name] | |
117 if isinstance(value, str): | |
118 ext_attrs_node[type_valued_attribute_name] = strip_modules(value) | |
119 | |
120 map(rename_node, idl_file.all(IDLInterface)) | |
121 map(rename_node, idl_file.all(IDLType)) | |
122 map(rename_ext_attrs, idl_file.all(IDLExtAttrs)) | |
123 | |
124 def _annotate(self, interface, import_options): | |
125 """Adds @ annotations based on the source and source_attributes | |
126 members of import_options.""" | |
127 | |
128 source = import_options.source | |
129 if not source: | |
130 return | |
131 | |
132 def add_source_annotation(idl_node): | |
133 annotation = IDLAnnotation( | |
134 copy.deepcopy(import_options.source_attributes)) | |
135 idl_node.annotations[source] = annotation | |
136 if ((isinstance(idl_node, IDLInterface) or | |
137 isinstance(idl_node, IDLMember)) and | |
138 idl_node.is_fc_suppressed): | |
139 annotation['suppressed'] = None | |
140 | |
141 add_source_annotation(interface) | |
142 | |
143 map(add_source_annotation, interface.parents) | |
144 map(add_source_annotation, interface.constants) | |
145 map(add_source_annotation, interface.attributes) | |
146 map(add_source_annotation, interface.operations) | |
147 | |
148 def _sign(self, node): | |
149 """Computes a unique signature for the node, for merging purposed, by | |
150 concatenating types and names in the declaration.""" | |
151 if isinstance(node, IDLType): | |
152 res = node.id | |
153 if res.startswith('unsigned '): | |
154 res = res[len('unsigned '):] | |
155 return res | |
156 | |
157 res = [] | |
158 if isinstance(node, IDLInterface): | |
159 res = ['interface', node.id] | |
160 elif isinstance(node, IDLParentInterface): | |
161 res = ['parent', self._sign(node.type)] | |
162 elif isinstance(node, IDLOperation): | |
163 res = ['op'] | |
164 for special in node.specials: | |
165 res.append(special) | |
166 if node.id is not None: | |
167 res.append(node.id) | |
168 for arg in node.arguments: | |
169 res.append(self._sign(arg.type)) | |
170 res.append(self._sign(node.type)) | |
171 elif isinstance(node, IDLAttribute): | |
172 res = [] | |
173 if node.is_read_only: | |
174 res.append('readonly') | |
175 res.append(node.id) | |
176 res.append(self._sign(node.type)) | |
177 elif isinstance(node, IDLConstant): | |
178 res = [] | |
179 res.append('const') | |
180 res.append(node.id) | |
181 res.append(node.value) | |
182 res.append(self._sign(node.type)) | |
183 else: | |
184 raise TypeError("Can't sign input of type %s" % type(node)) | |
185 return ':'.join(res) | |
186 | |
187 def _build_signatures_map(self, idl_node_list): | |
188 """Creates a hash table mapping signatures to idl_nodes for the | |
189 given list of nodes""" | |
190 res = {} | |
191 for idl_node in idl_node_list: | |
192 sig = self._sign(idl_node) | |
193 if sig is None: | |
194 continue | |
195 if sig in res: | |
196 raise RuntimeError('Warning: Multiple members have the same ' | |
197 'signature: "%s"' % sig) | |
198 res[sig] = idl_node | |
199 return res | |
200 | |
201 def _get_parent_interfaces(self, interface): | |
202 """Return a list of all the parent interfaces of a given interface""" | |
203 res = [] | |
204 | |
205 def recurse(current_interface): | |
206 if current_interface in res: | |
207 return | |
208 res.append(current_interface) | |
209 for parent in current_interface.parents: | |
210 parent_name = parent.type.id | |
211 if self._database.HasInterface(parent_name): | |
212 recurse(self._database.GetInterface(parent_name)) | |
213 | |
214 recurse(interface) | |
215 return res[1:] | |
216 | |
217 def _merge_ext_attrs(self, old_attrs, new_attrs): | |
218 """Merges two sets of extended attributes. | |
219 | |
220 Returns: True if old_attrs has changed. | |
221 """ | |
222 changed = False | |
223 for (name, value) in new_attrs.items(): | |
224 if name in old_attrs and old_attrs[name] == value: | |
225 pass # Identical | |
226 else: | |
227 old_attrs[name] = value | |
228 changed = True | |
229 return changed | |
230 | |
231 def _merge_nodes(self, old_list, new_list, import_options): | |
232 """Merges two lists of nodes. Annotates nodes with the source of each | |
233 node. | |
234 | |
235 Returns: | |
236 True if the old_list has changed. | |
237 | |
238 Args: | |
239 old_list -- the list to merge into. | |
240 new_list -- list containing more nodes. | |
241 import_options -- controls how merging is done. | |
242 """ | |
243 changed = False | |
244 | |
245 source = import_options.source | |
246 | |
247 old_signatures_map = self._build_signatures_map(old_list) | |
248 new_signatures_map = self._build_signatures_map(new_list) | |
249 | |
250 # Merge new items | |
251 for (sig, new_node) in new_signatures_map.items(): | |
252 if sig not in old_signatures_map: | |
253 # New node: | |
254 old_list.append(new_node) | |
255 changed = True | |
256 else: | |
257 # Merge old and new nodes: | |
258 old_node = old_signatures_map[sig] | |
259 if (source not in old_node.annotations | |
260 and source in new_node.annotations): | |
261 old_node.annotations[source] = new_node.annotations[source] | |
262 changed = True | |
263 # Maybe rename arguments: | |
264 if isinstance(old_node, IDLOperation): | |
265 for i in range(0, len(old_node.arguments)): | |
266 old_arg = old_node.arguments[i] | |
267 new_arg = new_node.arguments[i] | |
268 | |
269 old_arg_name = old_arg.id | |
270 new_arg_name = new_arg.id | |
271 if (old_arg_name != new_arg_name | |
272 and (old_arg_name == 'arg' | |
273 or old_arg_name.endswith('Arg') | |
274 or import_options.rename_operation_arguments_on_merge)): | |
275 old_node.arguments[i].id = new_arg_name | |
276 changed = True | |
277 | |
278 if self._merge_ext_attrs(old_arg.ext_attrs, new_arg.ext_attrs): | |
279 changed = True | |
280 # Maybe merge annotations: | |
281 if (isinstance(old_node, IDLAttribute) or | |
282 isinstance(old_node, IDLOperation)): | |
283 if self._merge_ext_attrs(old_node.ext_attrs, new_node.ext_attrs): | |
284 changed = True | |
285 | |
286 # Remove annotations on obsolete items from the same source | |
287 if import_options.obsolete_old_declarations: | |
288 for (sig, old_node) in old_signatures_map.items(): | |
289 if (source in old_node.annotations | |
290 and sig not in new_signatures_map): | |
291 _logger.warn('%s not available in %s anymore' % | |
292 (sig, source)) | |
293 del old_node.annotations[source] | |
294 changed = True | |
295 | |
296 return changed | |
297 | |
298 def _merge_interfaces(self, old_interface, new_interface, import_options): | |
299 """Merges the new_interface into the old_interface, annotating the | |
300 interface with the sources of each change.""" | |
301 | |
302 changed = False | |
303 | |
304 source = import_options.source | |
305 if (source and source not in old_interface.annotations and | |
306 source in new_interface.annotations and | |
307 not new_interface.is_supplemental): | |
308 old_interface.annotations[source] = new_interface.annotations[source] | |
309 changed = True | |
310 | |
311 def merge_list(what): | |
312 old_list = old_interface.__dict__[what] | |
313 new_list = new_interface.__dict__[what] | |
314 | |
315 if what != 'parents' and old_interface.id != new_interface.id: | |
316 for node in new_list: | |
317 node.doc_js_interface_name = old_interface.id | |
318 node.ext_attrs['ImplementedBy'] = new_interface.id | |
319 | |
320 changed = self._merge_nodes(old_list, new_list, import_options) | |
321 | |
322 # Delete list items with zero remaining annotations. | |
323 if changed and import_options.obsolete_old_declarations: | |
324 | |
325 def has_annotations(idl_node): | |
326 return len(idl_node.annotations) | |
327 | |
328 old_interface.__dict__[what] = filter(has_annotations, old_list) | |
329 | |
330 return changed | |
331 | |
332 # Smartly merge various declarations: | |
333 if merge_list('parents'): | |
334 changed = True | |
335 if merge_list('constants'): | |
336 changed = True | |
337 if merge_list('attributes'): | |
338 changed = True | |
339 if merge_list('operations'): | |
340 changed = True | |
341 | |
342 if self._merge_ext_attrs(old_interface.ext_attrs, new_interface.ext_attrs): | |
343 changed = True | |
344 | |
345 _logger.info('merged interface %s (changed=%s, supplemental=%s)' % | |
346 (old_interface.id, changed, new_interface.is_supplemental)) | |
347 | |
348 return changed | |
349 | |
350 def _merge_impl_stmt(self, impl_stmt, import_options): | |
351 """Applies "X implements Y" statemetns on the proper places in the | |
352 database""" | |
353 implementor_name = impl_stmt.implementor.id | |
354 implemented_name = impl_stmt.implemented.id | |
355 _logger.info('merging impl stmt %s implements %s' % | |
356 (implementor_name, implemented_name)) | |
357 | |
358 source = import_options.source | |
359 if self._database.HasInterface(implementor_name): | |
360 interface = self._database.GetInterface(implementor_name) | |
361 if interface.parents is None: | |
362 interface.parents = [] | |
363 for parent in interface.parents: | |
364 if parent.type.id == implemented_name: | |
365 if source and source not in parent.annotations: | |
366 parent.annotations[source] = IDLAnnotation( | |
367 import_options.source_attributes) | |
368 return | |
369 # not found, so add new one | |
370 parent = IDLParentInterface(None) | |
371 parent.type = IDLType(implemented_name) | |
372 if source: | |
373 parent.annotations[source] = IDLAnnotation( | |
374 import_options.source_attributes) | |
375 interface.parents.append(parent) | |
376 | |
377 def merge_imported_interfaces(self): | |
378 """Merges all imported interfaces and loads them into the DB.""" | |
379 | |
380 # Step 1: Pre process imported interfaces | |
381 for interface, import_options in self._imported_interfaces: | |
382 self._annotate(interface, import_options) | |
383 | |
384 # Step 2: Add all new interfaces and merge overlapping ones | |
385 for interface, import_options in self._imported_interfaces: | |
386 if not interface.is_supplemental: | |
387 if self._database.HasInterface(interface.id): | |
388 old_interface = self._database.GetInterface(interface.id) | |
389 self._merge_interfaces(old_interface, interface, import_options) | |
390 else: | |
391 if import_options.add_new_interfaces: | |
392 self._database.AddInterface(interface) | |
393 | |
394 # Step 3: Merge in supplemental interfaces | |
395 for interface, import_options in self._imported_interfaces: | |
396 if interface.is_supplemental: | |
397 target_name = interface.ext_attrs['Supplemental'] | |
398 if target_name: | |
399 # [Supplemental=DOMWindow] - merge into DOMWindow. | |
400 target = target_name | |
401 else: | |
402 # [Supplemental] - merge into existing inteface with same name. | |
403 target = interface.id | |
404 if self._database.HasInterface(target): | |
405 old_interface = self._database.GetInterface(target) | |
406 self._merge_interfaces(old_interface, interface, import_options) | |
407 else: | |
408 raise Exception("Supplemental target '%s' not found", target) | |
409 | |
410 # Step 4: Resolve 'implements' statements | |
411 for impl_stmt, import_options in self._impl_stmts: | |
412 self._merge_impl_stmt(impl_stmt, import_options) | |
413 | |
414 self._impl_stmts = [] | |
415 self._imported_interfaces = [] | |
416 | |
417 def import_idl_files(self, file_paths, import_options, parallel): | |
418 if parallel: | |
419 # Parse the IDL files in parallel. | |
420 pool = multiprocessing.Pool() | |
421 try: | |
422 for file_path in file_paths: | |
423 pool.apply_async(_load_idl_file, | |
424 [ file_path, import_options], | |
425 callback = lambda idl_file: | |
426 self._process_idl_file(idl_file, import_options)) | |
427 pool.close() | |
428 pool.join() | |
429 except: | |
430 pool.terminate() | |
431 raise | |
432 else: | |
433 # Parse the IDL files in serial. | |
434 for file_path in file_paths: | |
435 idl_file = _load_idl_file(file_path, import_options) | |
436 self._process_idl_file(idl_file, import_options) | |
437 | |
438 def _process_idl_file(self, idl_file, | |
439 import_options): | |
440 self._strip_ext_attributes(idl_file) | |
441 self._resolve_type_defs(idl_file) | |
442 self._rename_types(idl_file, import_options) | |
443 | |
444 def enabled(idl_node): | |
445 return self._is_node_enabled(idl_node, import_options.idl_defines) | |
446 | |
447 for interface in idl_file.interfaces: | |
448 if not self._is_node_enabled(interface, import_options.idl_defines): | |
449 _logger.info('skipping interface %s (source=%s)' | |
450 % (interface.id, import_options.source)) | |
451 continue | |
452 | |
453 _logger.info('importing interface %s (source=%s)' | |
454 % (interface.id, import_options.source)) | |
455 interface.attributes = filter(enabled, interface.attributes) | |
456 interface.operations = filter(enabled, interface.operations) | |
457 self._imported_interfaces.append((interface, import_options)) | |
458 | |
459 for implStmt in idl_file.implementsStatements: | |
460 self._impl_stmts.append((implStmt, import_options)) | |
461 | |
462 | |
463 def _is_node_enabled(self, node, idl_defines): | |
464 if not 'Conditional' in node.ext_attrs: | |
465 return True | |
466 | |
467 def enabled(condition): | |
468 return 'ENABLE_%s' % condition in idl_defines | |
469 | |
470 conditional = node.ext_attrs['Conditional'] | |
471 if conditional.find('&') != -1: | |
472 for condition in conditional.split('&'): | |
473 if not enabled(condition): | |
474 return False | |
475 return True | |
476 | |
477 for condition in conditional.split('|'): | |
478 if enabled(condition): | |
479 return True | |
480 return False | |
481 | |
482 def fix_displacements(self, source): | |
483 """E.g. In W3C, something is declared on HTMLDocument but in WebKit | |
484 its on Document, so we need to mark that something in HTMLDocument | |
485 with @WebKit(via=Document). The 'via' attribute specifies the | |
486 parent interface that has the declaration.""" | |
487 | |
488 for interface in self._database.GetInterfaces(): | |
489 changed = False | |
490 | |
491 _logger.info('fixing displacements in %s' % interface.id) | |
492 | |
493 for parent_interface in self._get_parent_interfaces(interface): | |
494 _logger.info('scanning parent %s of %s' % | |
495 (parent_interface.id, interface.id)) | |
496 | |
497 def fix_nodes(local_list, parent_list): | |
498 changed = False | |
499 parent_signatures_map = self._build_signatures_map( | |
500 parent_list) | |
501 for idl_node in local_list: | |
502 sig = self._sign(idl_node) | |
503 if sig in parent_signatures_map: | |
504 parent_member = parent_signatures_map[sig] | |
505 if (source in parent_member.annotations | |
506 and source not in idl_node.annotations | |
507 and _VIA_ANNOTATION_ATTR_NAME | |
508 not in parent_member.annotations[source]): | |
509 idl_node.annotations[source] = IDLAnnotation( | |
510 {_VIA_ANNOTATION_ATTR_NAME: parent_interface.id}) | |
511 changed = True | |
512 return changed | |
513 | |
514 changed = fix_nodes(interface.constants, | |
515 parent_interface.constants) or changed | |
516 changed = fix_nodes(interface.attributes, | |
517 parent_interface.attributes) or changed | |
518 changed = fix_nodes(interface.operations, | |
519 parent_interface.operations) or changed | |
520 if changed: | |
521 _logger.info('fixed displaced declarations in %s' % | |
522 interface.id) | |
523 | |
524 def normalize_annotations(self, sources): | |
525 """Makes the IDLs less verbose by removing annotation attributes | |
526 that are identical to the ones defined at the interface level. | |
527 | |
528 Args: | |
529 sources -- list of source names to normalize.""" | |
530 for interface in self._database.GetInterfaces(): | |
531 _logger.debug('normalizing annotations for %s' % interface.id) | |
532 for source in sources: | |
533 if (source not in interface.annotations or | |
534 not interface.annotations[source]): | |
535 continue | |
536 top_level_annotation = interface.annotations[source] | |
537 | |
538 def normalize(idl_node): | |
539 if (source in idl_node.annotations | |
540 and idl_node.annotations[source]): | |
541 annotation = idl_node.annotations[source] | |
542 for name, value in annotation.items(): | |
543 if (name in top_level_annotation | |
544 and value == top_level_annotation[name]): | |
545 del annotation[name] | |
546 | |
547 map(normalize, interface.parents) | |
548 map(normalize, interface.constants) | |
549 map(normalize, interface.attributes) | |
550 map(normalize, interface.operations) | |
551 | |
552 def fetch_constructor_data(self, options): | |
553 window_interface = self._database.GetInterface('DOMWindow') | |
554 for attr in window_interface.attributes: | |
555 type = attr.type.id | |
556 if not type.endswith('Constructor'): | |
557 continue | |
558 type = re.sub('(Constructor)+$', '', type) | |
559 # TODO(antonm): Ideally we'd like to have pristine copy of WebKit IDLs and
fetch | |
560 # this information directly from it. Unfortunately right now database is
massaged | |
561 # a lot so it's difficult to maintain necessary information on DOMWindow i
tself. | |
562 interface = self._database.GetInterface(type) | |
563 if 'V8EnabledPerContext' in attr.ext_attrs: | |
564 interface.ext_attrs['synthesizedV8EnabledPerContext'] = \ | |
565 attr.ext_attrs['V8EnabledPerContext'] | |
566 if 'V8EnabledAtRuntime' in attr.ext_attrs: | |
567 interface.ext_attrs['synthesizedV8EnabledAtRuntime'] = \ | |
568 attr.ext_attrs['V8EnabledAtRuntime'] or attr.id | |
OLD | NEW |