Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(111)

Side by Side Diff: sdk/lib/html/scripts/databasebuilder.py

Issue 11691009: Moved most of html lib generating scripts into tools. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 7 years, 12 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « sdk/lib/html/scripts/database_test.py ('k') | sdk/lib/html/scripts/databasebuilder_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/python
2 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
3 # for details. All rights reserved. Use of this source code is governed by a
4 # BSD-style license that can be found in the LICENSE file.
5
6 import copy
7 import database
8 import idlparser
9 import logging
10 import multiprocessing
11 import os
12 import os.path
13 import re
14
15 from idlnode import *
16
17 _logger = logging.getLogger('databasebuilder')
18
19 # Used in source annotations to specify the parent interface declaring
20 # a displaced declaration. The 'via' attribute specifies the parent interface
21 # which implements a displaced declaration.
22 _VIA_ANNOTATION_ATTR_NAME = 'via'
23
24
25 class DatabaseBuilderOptions(object):
26 """Used in specifying options when importing new interfaces"""
27
28 def __init__(self,
29 idl_syntax=idlparser.WEBIDL_SYNTAX,
30 idl_defines=[],
31 source=None, source_attributes={},
32 rename_operation_arguments_on_merge=False,
33 add_new_interfaces=True,
34 obsolete_old_declarations=False):
35 """Constructor.
36 Args:
37 idl_syntax -- the syntax of the IDL file that is imported.
38 idl_defines -- list of definitions for the idl gcc pre-processor
39 source -- the origin of the IDL file, used for annotating the
40 database.
41 source_attributes -- this map of attributes is used as
42 annotation attributes.
43 rename_operation_arguments_on_merge -- if True, will rename
44 operation arguments when merging using the new name rather
45 than the old.
46 add_new_interfaces -- when False, if an interface is a new
47 addition, it will be ignored.
48 obsolete_old_declarations -- when True, if a declaration
49 from a certain source is not re-declared, it will be removed.
50 """
51 self.source = source
52 self.source_attributes = source_attributes
53 self.idl_syntax = idl_syntax
54 self.idl_defines = idl_defines
55 self.rename_operation_arguments_on_merge = \
56 rename_operation_arguments_on_merge
57 self.add_new_interfaces = add_new_interfaces
58 self.obsolete_old_declarations = obsolete_old_declarations
59
60
61 def _load_idl_file(file_name, import_options):
62 """Loads an IDL file into memory"""
63 idl_parser = idlparser.IDLParser(import_options.idl_syntax)
64
65 try:
66 f = open(file_name, 'r')
67 content = f.read()
68 f.close()
69
70 idl_ast = idl_parser.parse(
71 content,
72 defines=import_options.idl_defines)
73 return IDLFile(idl_ast, file_name)
74 except SyntaxError, e:
75 raise RuntimeError('Failed to load file %s: %s'
76 % (file_name, e))
77
78
79 class DatabaseBuilder(object):
80 def __init__(self, database):
81 """DatabaseBuilder is used for importing and merging interfaces into
82 the Database"""
83 self._database = database
84 self._imported_interfaces = []
85 self._impl_stmts = []
86
87 def _resolve_type_defs(self, idl_file):
88 type_def_map = {}
89 # build map
90 for type_def in idl_file.all(IDLTypeDef):
91 if type_def.type.id != type_def.id: # sanity check
92 type_def_map[type_def.id] = type_def.type.id
93 # use the map
94 for type_node in idl_file.all(IDLType):
95 while type_node.id in type_def_map:
96 type_node.id = type_def_map[type_node.id]
97
98 def _strip_ext_attributes(self, idl_file):
99 """Strips unuseful extended attributes."""
100 for ext_attrs in idl_file.all(IDLExtAttrs):
101 # TODO: Decide which attributes are uninteresting.
102 pass
103
104 def _rename_types(self, idl_file, import_options):
105 """Rename interface and type names with names provided in the
106 options. Also clears scopes from scoped names"""
107
108 strip_modules = lambda name: name.split('::')[-1]
109
110 def rename_node(idl_node):
111 idl_node.reset_id(strip_modules(idl_node.id))
112
113 def rename_ext_attrs(ext_attrs_node):
114 for type_valued_attribute_name in ['Supplemental']:
115 if type_valued_attribute_name in ext_attrs_node:
116 value = ext_attrs_node[type_valued_attribute_name]
117 if isinstance(value, str):
118 ext_attrs_node[type_valued_attribute_name] = strip_modules(value)
119
120 map(rename_node, idl_file.all(IDLInterface))
121 map(rename_node, idl_file.all(IDLType))
122 map(rename_ext_attrs, idl_file.all(IDLExtAttrs))
123
124 def _annotate(self, interface, import_options):
125 """Adds @ annotations based on the source and source_attributes
126 members of import_options."""
127
128 source = import_options.source
129 if not source:
130 return
131
132 def add_source_annotation(idl_node):
133 annotation = IDLAnnotation(
134 copy.deepcopy(import_options.source_attributes))
135 idl_node.annotations[source] = annotation
136 if ((isinstance(idl_node, IDLInterface) or
137 isinstance(idl_node, IDLMember)) and
138 idl_node.is_fc_suppressed):
139 annotation['suppressed'] = None
140
141 add_source_annotation(interface)
142
143 map(add_source_annotation, interface.parents)
144 map(add_source_annotation, interface.constants)
145 map(add_source_annotation, interface.attributes)
146 map(add_source_annotation, interface.operations)
147
148 def _sign(self, node):
149 """Computes a unique signature for the node, for merging purposed, by
150 concatenating types and names in the declaration."""
151 if isinstance(node, IDLType):
152 res = node.id
153 if res.startswith('unsigned '):
154 res = res[len('unsigned '):]
155 return res
156
157 res = []
158 if isinstance(node, IDLInterface):
159 res = ['interface', node.id]
160 elif isinstance(node, IDLParentInterface):
161 res = ['parent', self._sign(node.type)]
162 elif isinstance(node, IDLOperation):
163 res = ['op']
164 for special in node.specials:
165 res.append(special)
166 if node.id is not None:
167 res.append(node.id)
168 for arg in node.arguments:
169 res.append(self._sign(arg.type))
170 res.append(self._sign(node.type))
171 elif isinstance(node, IDLAttribute):
172 res = []
173 if node.is_read_only:
174 res.append('readonly')
175 res.append(node.id)
176 res.append(self._sign(node.type))
177 elif isinstance(node, IDLConstant):
178 res = []
179 res.append('const')
180 res.append(node.id)
181 res.append(node.value)
182 res.append(self._sign(node.type))
183 else:
184 raise TypeError("Can't sign input of type %s" % type(node))
185 return ':'.join(res)
186
187 def _build_signatures_map(self, idl_node_list):
188 """Creates a hash table mapping signatures to idl_nodes for the
189 given list of nodes"""
190 res = {}
191 for idl_node in idl_node_list:
192 sig = self._sign(idl_node)
193 if sig is None:
194 continue
195 if sig in res:
196 raise RuntimeError('Warning: Multiple members have the same '
197 'signature: "%s"' % sig)
198 res[sig] = idl_node
199 return res
200
201 def _get_parent_interfaces(self, interface):
202 """Return a list of all the parent interfaces of a given interface"""
203 res = []
204
205 def recurse(current_interface):
206 if current_interface in res:
207 return
208 res.append(current_interface)
209 for parent in current_interface.parents:
210 parent_name = parent.type.id
211 if self._database.HasInterface(parent_name):
212 recurse(self._database.GetInterface(parent_name))
213
214 recurse(interface)
215 return res[1:]
216
217 def _merge_ext_attrs(self, old_attrs, new_attrs):
218 """Merges two sets of extended attributes.
219
220 Returns: True if old_attrs has changed.
221 """
222 changed = False
223 for (name, value) in new_attrs.items():
224 if name in old_attrs and old_attrs[name] == value:
225 pass # Identical
226 else:
227 old_attrs[name] = value
228 changed = True
229 return changed
230
231 def _merge_nodes(self, old_list, new_list, import_options):
232 """Merges two lists of nodes. Annotates nodes with the source of each
233 node.
234
235 Returns:
236 True if the old_list has changed.
237
238 Args:
239 old_list -- the list to merge into.
240 new_list -- list containing more nodes.
241 import_options -- controls how merging is done.
242 """
243 changed = False
244
245 source = import_options.source
246
247 old_signatures_map = self._build_signatures_map(old_list)
248 new_signatures_map = self._build_signatures_map(new_list)
249
250 # Merge new items
251 for (sig, new_node) in new_signatures_map.items():
252 if sig not in old_signatures_map:
253 # New node:
254 old_list.append(new_node)
255 changed = True
256 else:
257 # Merge old and new nodes:
258 old_node = old_signatures_map[sig]
259 if (source not in old_node.annotations
260 and source in new_node.annotations):
261 old_node.annotations[source] = new_node.annotations[source]
262 changed = True
263 # Maybe rename arguments:
264 if isinstance(old_node, IDLOperation):
265 for i in range(0, len(old_node.arguments)):
266 old_arg = old_node.arguments[i]
267 new_arg = new_node.arguments[i]
268
269 old_arg_name = old_arg.id
270 new_arg_name = new_arg.id
271 if (old_arg_name != new_arg_name
272 and (old_arg_name == 'arg'
273 or old_arg_name.endswith('Arg')
274 or import_options.rename_operation_arguments_on_merge)):
275 old_node.arguments[i].id = new_arg_name
276 changed = True
277
278 if self._merge_ext_attrs(old_arg.ext_attrs, new_arg.ext_attrs):
279 changed = True
280 # Maybe merge annotations:
281 if (isinstance(old_node, IDLAttribute) or
282 isinstance(old_node, IDLOperation)):
283 if self._merge_ext_attrs(old_node.ext_attrs, new_node.ext_attrs):
284 changed = True
285
286 # Remove annotations on obsolete items from the same source
287 if import_options.obsolete_old_declarations:
288 for (sig, old_node) in old_signatures_map.items():
289 if (source in old_node.annotations
290 and sig not in new_signatures_map):
291 _logger.warn('%s not available in %s anymore' %
292 (sig, source))
293 del old_node.annotations[source]
294 changed = True
295
296 return changed
297
298 def _merge_interfaces(self, old_interface, new_interface, import_options):
299 """Merges the new_interface into the old_interface, annotating the
300 interface with the sources of each change."""
301
302 changed = False
303
304 source = import_options.source
305 if (source and source not in old_interface.annotations and
306 source in new_interface.annotations and
307 not new_interface.is_supplemental):
308 old_interface.annotations[source] = new_interface.annotations[source]
309 changed = True
310
311 def merge_list(what):
312 old_list = old_interface.__dict__[what]
313 new_list = new_interface.__dict__[what]
314
315 if what != 'parents' and old_interface.id != new_interface.id:
316 for node in new_list:
317 node.doc_js_interface_name = old_interface.id
318 node.ext_attrs['ImplementedBy'] = new_interface.id
319
320 changed = self._merge_nodes(old_list, new_list, import_options)
321
322 # Delete list items with zero remaining annotations.
323 if changed and import_options.obsolete_old_declarations:
324
325 def has_annotations(idl_node):
326 return len(idl_node.annotations)
327
328 old_interface.__dict__[what] = filter(has_annotations, old_list)
329
330 return changed
331
332 # Smartly merge various declarations:
333 if merge_list('parents'):
334 changed = True
335 if merge_list('constants'):
336 changed = True
337 if merge_list('attributes'):
338 changed = True
339 if merge_list('operations'):
340 changed = True
341
342 if self._merge_ext_attrs(old_interface.ext_attrs, new_interface.ext_attrs):
343 changed = True
344
345 _logger.info('merged interface %s (changed=%s, supplemental=%s)' %
346 (old_interface.id, changed, new_interface.is_supplemental))
347
348 return changed
349
350 def _merge_impl_stmt(self, impl_stmt, import_options):
351 """Applies "X implements Y" statemetns on the proper places in the
352 database"""
353 implementor_name = impl_stmt.implementor.id
354 implemented_name = impl_stmt.implemented.id
355 _logger.info('merging impl stmt %s implements %s' %
356 (implementor_name, implemented_name))
357
358 source = import_options.source
359 if self._database.HasInterface(implementor_name):
360 interface = self._database.GetInterface(implementor_name)
361 if interface.parents is None:
362 interface.parents = []
363 for parent in interface.parents:
364 if parent.type.id == implemented_name:
365 if source and source not in parent.annotations:
366 parent.annotations[source] = IDLAnnotation(
367 import_options.source_attributes)
368 return
369 # not found, so add new one
370 parent = IDLParentInterface(None)
371 parent.type = IDLType(implemented_name)
372 if source:
373 parent.annotations[source] = IDLAnnotation(
374 import_options.source_attributes)
375 interface.parents.append(parent)
376
377 def merge_imported_interfaces(self):
378 """Merges all imported interfaces and loads them into the DB."""
379
380 # Step 1: Pre process imported interfaces
381 for interface, import_options in self._imported_interfaces:
382 self._annotate(interface, import_options)
383
384 # Step 2: Add all new interfaces and merge overlapping ones
385 for interface, import_options in self._imported_interfaces:
386 if not interface.is_supplemental:
387 if self._database.HasInterface(interface.id):
388 old_interface = self._database.GetInterface(interface.id)
389 self._merge_interfaces(old_interface, interface, import_options)
390 else:
391 if import_options.add_new_interfaces:
392 self._database.AddInterface(interface)
393
394 # Step 3: Merge in supplemental interfaces
395 for interface, import_options in self._imported_interfaces:
396 if interface.is_supplemental:
397 target_name = interface.ext_attrs['Supplemental']
398 if target_name:
399 # [Supplemental=DOMWindow] - merge into DOMWindow.
400 target = target_name
401 else:
402 # [Supplemental] - merge into existing inteface with same name.
403 target = interface.id
404 if self._database.HasInterface(target):
405 old_interface = self._database.GetInterface(target)
406 self._merge_interfaces(old_interface, interface, import_options)
407 else:
408 raise Exception("Supplemental target '%s' not found", target)
409
410 # Step 4: Resolve 'implements' statements
411 for impl_stmt, import_options in self._impl_stmts:
412 self._merge_impl_stmt(impl_stmt, import_options)
413
414 self._impl_stmts = []
415 self._imported_interfaces = []
416
417 def import_idl_files(self, file_paths, import_options, parallel):
418 if parallel:
419 # Parse the IDL files in parallel.
420 pool = multiprocessing.Pool()
421 try:
422 for file_path in file_paths:
423 pool.apply_async(_load_idl_file,
424 [ file_path, import_options],
425 callback = lambda idl_file:
426 self._process_idl_file(idl_file, import_options))
427 pool.close()
428 pool.join()
429 except:
430 pool.terminate()
431 raise
432 else:
433 # Parse the IDL files in serial.
434 for file_path in file_paths:
435 idl_file = _load_idl_file(file_path, import_options)
436 self._process_idl_file(idl_file, import_options)
437
438 def _process_idl_file(self, idl_file,
439 import_options):
440 self._strip_ext_attributes(idl_file)
441 self._resolve_type_defs(idl_file)
442 self._rename_types(idl_file, import_options)
443
444 def enabled(idl_node):
445 return self._is_node_enabled(idl_node, import_options.idl_defines)
446
447 for interface in idl_file.interfaces:
448 if not self._is_node_enabled(interface, import_options.idl_defines):
449 _logger.info('skipping interface %s (source=%s)'
450 % (interface.id, import_options.source))
451 continue
452
453 _logger.info('importing interface %s (source=%s)'
454 % (interface.id, import_options.source))
455 interface.attributes = filter(enabled, interface.attributes)
456 interface.operations = filter(enabled, interface.operations)
457 self._imported_interfaces.append((interface, import_options))
458
459 for implStmt in idl_file.implementsStatements:
460 self._impl_stmts.append((implStmt, import_options))
461
462
463 def _is_node_enabled(self, node, idl_defines):
464 if not 'Conditional' in node.ext_attrs:
465 return True
466
467 def enabled(condition):
468 return 'ENABLE_%s' % condition in idl_defines
469
470 conditional = node.ext_attrs['Conditional']
471 if conditional.find('&') != -1:
472 for condition in conditional.split('&'):
473 if not enabled(condition):
474 return False
475 return True
476
477 for condition in conditional.split('|'):
478 if enabled(condition):
479 return True
480 return False
481
482 def fix_displacements(self, source):
483 """E.g. In W3C, something is declared on HTMLDocument but in WebKit
484 its on Document, so we need to mark that something in HTMLDocument
485 with @WebKit(via=Document). The 'via' attribute specifies the
486 parent interface that has the declaration."""
487
488 for interface in self._database.GetInterfaces():
489 changed = False
490
491 _logger.info('fixing displacements in %s' % interface.id)
492
493 for parent_interface in self._get_parent_interfaces(interface):
494 _logger.info('scanning parent %s of %s' %
495 (parent_interface.id, interface.id))
496
497 def fix_nodes(local_list, parent_list):
498 changed = False
499 parent_signatures_map = self._build_signatures_map(
500 parent_list)
501 for idl_node in local_list:
502 sig = self._sign(idl_node)
503 if sig in parent_signatures_map:
504 parent_member = parent_signatures_map[sig]
505 if (source in parent_member.annotations
506 and source not in idl_node.annotations
507 and _VIA_ANNOTATION_ATTR_NAME
508 not in parent_member.annotations[source]):
509 idl_node.annotations[source] = IDLAnnotation(
510 {_VIA_ANNOTATION_ATTR_NAME: parent_interface.id})
511 changed = True
512 return changed
513
514 changed = fix_nodes(interface.constants,
515 parent_interface.constants) or changed
516 changed = fix_nodes(interface.attributes,
517 parent_interface.attributes) or changed
518 changed = fix_nodes(interface.operations,
519 parent_interface.operations) or changed
520 if changed:
521 _logger.info('fixed displaced declarations in %s' %
522 interface.id)
523
524 def normalize_annotations(self, sources):
525 """Makes the IDLs less verbose by removing annotation attributes
526 that are identical to the ones defined at the interface level.
527
528 Args:
529 sources -- list of source names to normalize."""
530 for interface in self._database.GetInterfaces():
531 _logger.debug('normalizing annotations for %s' % interface.id)
532 for source in sources:
533 if (source not in interface.annotations or
534 not interface.annotations[source]):
535 continue
536 top_level_annotation = interface.annotations[source]
537
538 def normalize(idl_node):
539 if (source in idl_node.annotations
540 and idl_node.annotations[source]):
541 annotation = idl_node.annotations[source]
542 for name, value in annotation.items():
543 if (name in top_level_annotation
544 and value == top_level_annotation[name]):
545 del annotation[name]
546
547 map(normalize, interface.parents)
548 map(normalize, interface.constants)
549 map(normalize, interface.attributes)
550 map(normalize, interface.operations)
551
552 def fetch_constructor_data(self, options):
553 window_interface = self._database.GetInterface('DOMWindow')
554 for attr in window_interface.attributes:
555 type = attr.type.id
556 if not type.endswith('Constructor'):
557 continue
558 type = re.sub('(Constructor)+$', '', type)
559 # TODO(antonm): Ideally we'd like to have pristine copy of WebKit IDLs and fetch
560 # this information directly from it. Unfortunately right now database is massaged
561 # a lot so it's difficult to maintain necessary information on DOMWindow i tself.
562 interface = self._database.GetInterface(type)
563 if 'V8EnabledPerContext' in attr.ext_attrs:
564 interface.ext_attrs['synthesizedV8EnabledPerContext'] = \
565 attr.ext_attrs['V8EnabledPerContext']
566 if 'V8EnabledAtRuntime' in attr.ext_attrs:
567 interface.ext_attrs['synthesizedV8EnabledAtRuntime'] = \
568 attr.ext_attrs['V8EnabledAtRuntime'] or attr.id
OLDNEW
« no previous file with comments | « sdk/lib/html/scripts/database_test.py ('k') | sdk/lib/html/scripts/databasebuilder_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698