Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(437)

Side by Side Diff: third_party/closure_linter/closure_linter/closurizednamespacesinfo.py

Issue 2592193002: Remove closure_linter from Chrome (Closed)
Patch Set: Created 3 years, 12 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 #
3 # Copyright 2008 The Closure Linter Authors. All Rights Reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS-IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16
17 """Logic for computing dependency information for closurized JavaScript files.
18
19 Closurized JavaScript files express dependencies using goog.require and
20 goog.provide statements. In order for the linter to detect when a statement is
21 missing or unnecessary, all identifiers in the JavaScript file must first be
22 processed to determine if they constitute the creation or usage of a dependency.
23 """
24
25
26
27 import re
28
29 from closure_linter import javascripttokens
30 from closure_linter import tokenutil
31
32 # pylint: disable=g-bad-name
33 TokenType = javascripttokens.JavaScriptTokenType
34
35 DEFAULT_EXTRA_NAMESPACES = [
36 'goog.testing.asserts',
37 'goog.testing.jsunit',
38 ]
39
40
41 class UsedNamespace(object):
42 """A type for information about a used namespace."""
43
44 def __init__(self, namespace, identifier, token, alias_definition):
45 """Initializes the instance.
46
47 Args:
48 namespace: the namespace of an identifier used in the file
49 identifier: the complete identifier
50 token: the token that uses the namespace
51 alias_definition: a boolean stating whether the namespace is only to used
52 for an alias definition and should not be required.
53 """
54 self.namespace = namespace
55 self.identifier = identifier
56 self.token = token
57 self.alias_definition = alias_definition
58
59 def GetLine(self):
60 return self.token.line_number
61
62 def __repr__(self):
63 return 'UsedNamespace(%s)' % ', '.join(
64 ['%s=%s' % (k, repr(v)) for k, v in self.__dict__.iteritems()])
65
66
67 class ClosurizedNamespacesInfo(object):
68 """Dependency information for closurized JavaScript files.
69
70 Processes token streams for dependency creation or usage and provides logic
71 for determining if a given require or provide statement is unnecessary or if
72 there are missing require or provide statements.
73 """
74
75 def __init__(self, closurized_namespaces, ignored_extra_namespaces):
76 """Initializes an instance the ClosurizedNamespacesInfo class.
77
78 Args:
79 closurized_namespaces: A list of namespace prefixes that should be
80 processed for dependency information. Non-matching namespaces are
81 ignored.
82 ignored_extra_namespaces: A list of namespaces that should not be reported
83 as extra regardless of whether they are actually used.
84 """
85 self._closurized_namespaces = closurized_namespaces
86 self._ignored_extra_namespaces = (ignored_extra_namespaces +
87 DEFAULT_EXTRA_NAMESPACES)
88 self.Reset()
89
90 def Reset(self):
91 """Resets the internal state to prepare for processing a new file."""
92
93 # A list of goog.provide tokens in the order they appeared in the file.
94 self._provide_tokens = []
95
96 # A list of goog.require tokens in the order they appeared in the file.
97 self._require_tokens = []
98
99 # Namespaces that are already goog.provided.
100 self._provided_namespaces = []
101
102 # Namespaces that are already goog.required.
103 self._required_namespaces = []
104
105 # Note that created_namespaces and used_namespaces contain both namespaces
106 # and identifiers because there are many existing cases where a method or
107 # constant is provided directly instead of its namespace. Ideally, these
108 # two lists would only have to contain namespaces.
109
110 # A list of tuples where the first element is the namespace of an identifier
111 # created in the file, the second is the identifier itself and the third is
112 # the line number where it's created.
113 self._created_namespaces = []
114
115 # A list of UsedNamespace instances.
116 self._used_namespaces = []
117
118 # A list of seemingly-unnecessary namespaces that are goog.required() and
119 # annotated with @suppress {extraRequire}.
120 self._suppressed_requires = []
121
122 # A list of goog.provide tokens which are duplicates.
123 self._duplicate_provide_tokens = []
124
125 # A list of goog.require tokens which are duplicates.
126 self._duplicate_require_tokens = []
127
128 # Whether this file is in a goog.scope. Someday, we may add support
129 # for checking scopified namespaces, but for now let's just fail
130 # in a more reasonable way.
131 self._scopified_file = False
132
133 # TODO(user): Handle the case where there are 2 different requires
134 # that can satisfy the same dependency, but only one is necessary.
135
136 def GetProvidedNamespaces(self):
137 """Returns the namespaces which are already provided by this file.
138
139 Returns:
140 A list of strings where each string is a 'namespace' corresponding to an
141 existing goog.provide statement in the file being checked.
142 """
143 return set(self._provided_namespaces)
144
145 def GetRequiredNamespaces(self):
146 """Returns the namespaces which are already required by this file.
147
148 Returns:
149 A list of strings where each string is a 'namespace' corresponding to an
150 existing goog.require statement in the file being checked.
151 """
152 return set(self._required_namespaces)
153
154 def IsExtraProvide(self, token):
155 """Returns whether the given goog.provide token is unnecessary.
156
157 Args:
158 token: A goog.provide token.
159
160 Returns:
161 True if the given token corresponds to an unnecessary goog.provide
162 statement, otherwise False.
163 """
164 namespace = tokenutil.GetStringAfterToken(token)
165
166 if self.GetClosurizedNamespace(namespace) is None:
167 return False
168
169 if token in self._duplicate_provide_tokens:
170 return True
171
172 # TODO(user): There's probably a faster way to compute this.
173 for created_namespace, created_identifier, _ in self._created_namespaces:
174 if namespace == created_namespace or namespace == created_identifier:
175 return False
176
177 return True
178
179 def IsExtraRequire(self, token):
180 """Returns whether the given goog.require token is unnecessary.
181
182 Args:
183 token: A goog.require token.
184
185 Returns:
186 True if the given token corresponds to an unnecessary goog.require
187 statement, otherwise False.
188 """
189 namespace = tokenutil.GetStringAfterToken(token)
190
191 if self.GetClosurizedNamespace(namespace) is None:
192 return False
193
194 if namespace in self._ignored_extra_namespaces:
195 return False
196
197 if token in self._duplicate_require_tokens:
198 return True
199
200 if namespace in self._suppressed_requires:
201 return False
202
203 # If the namespace contains a component that is initial caps, then that
204 # must be the last component of the namespace.
205 parts = namespace.split('.')
206 if len(parts) > 1 and parts[-2][0].isupper():
207 return True
208
209 # TODO(user): There's probably a faster way to compute this.
210 for ns in self._used_namespaces:
211 if (not ns.alias_definition and (
212 namespace == ns.namespace or namespace == ns.identifier)):
213 return False
214
215 return True
216
217 def GetMissingProvides(self):
218 """Returns the dict of missing provided namespaces for the current file.
219
220 Returns:
221 Returns a dictionary of key as string and value as integer where each
222 string(key) is a namespace that should be provided by this file, but is
223 not and integer(value) is first line number where it's defined.
224 """
225 missing_provides = dict()
226 for namespace, identifier, line_number in self._created_namespaces:
227 if (not self._IsPrivateIdentifier(identifier) and
228 namespace not in self._provided_namespaces and
229 identifier not in self._provided_namespaces and
230 namespace not in self._required_namespaces and
231 namespace not in missing_provides):
232 missing_provides[namespace] = line_number
233
234 return missing_provides
235
236 def GetMissingRequires(self):
237 """Returns the dict of missing required namespaces for the current file.
238
239 For each non-private identifier used in the file, find either a
240 goog.require, goog.provide or a created identifier that satisfies it.
241 goog.require statements can satisfy the identifier by requiring either the
242 namespace of the identifier or the identifier itself. goog.provide
243 statements can satisfy the identifier by providing the namespace of the
244 identifier. A created identifier can only satisfy the used identifier if
245 it matches it exactly (necessary since things can be defined on a
246 namespace in more than one file). Note that provided namespaces should be
247 a subset of created namespaces, but we check both because in some cases we
248 can't always detect the creation of the namespace.
249
250 Returns:
251 Returns a dictionary of key as string and value integer where each
252 string(key) is a namespace that should be required by this file, but is
253 not and integer(value) is first line number where it's used.
254 """
255 external_dependencies = set(self._required_namespaces)
256
257 # Assume goog namespace is always available.
258 external_dependencies.add('goog')
259 # goog.module is treated as a builtin, too (for goog.module.get).
260 external_dependencies.add('goog.module')
261
262 created_identifiers = set()
263 for unused_namespace, identifier, unused_line_number in (
264 self._created_namespaces):
265 created_identifiers.add(identifier)
266
267 missing_requires = dict()
268 illegal_alias_statements = dict()
269
270 def ShouldRequireNamespace(namespace, identifier):
271 """Checks if a namespace would normally be required."""
272 return (
273 not self._IsPrivateIdentifier(identifier) and
274 namespace not in external_dependencies and
275 namespace not in self._provided_namespaces and
276 identifier not in external_dependencies and
277 identifier not in created_identifiers and
278 namespace not in missing_requires)
279
280 # First check all the used identifiers where we know that their namespace
281 # needs to be provided (unless they are optional).
282 for ns in self._used_namespaces:
283 namespace = ns.namespace
284 identifier = ns.identifier
285 if (not ns.alias_definition and
286 ShouldRequireNamespace(namespace, identifier)):
287 missing_requires[namespace] = ns.GetLine()
288
289 # Now that all required namespaces are known, we can check if the alias
290 # definitions (that are likely being used for typeannotations that don't
291 # need explicit goog.require statements) are already covered. If not
292 # the user shouldn't use the alias.
293 for ns in self._used_namespaces:
294 if (not ns.alias_definition or
295 not ShouldRequireNamespace(ns.namespace, ns.identifier)):
296 continue
297 if self._FindNamespace(ns.identifier, self._provided_namespaces,
298 created_identifiers, external_dependencies,
299 missing_requires):
300 continue
301 namespace = ns.identifier.rsplit('.', 1)[0]
302 illegal_alias_statements[namespace] = ns.token
303
304 return missing_requires, illegal_alias_statements
305
306 def _FindNamespace(self, identifier, *namespaces_list):
307 """Finds the namespace of an identifier given a list of other namespaces.
308
309 Args:
310 identifier: An identifier whose parent needs to be defined.
311 e.g. for goog.bar.foo we search something that provides
312 goog.bar.
313 *namespaces_list: var args of iterables of namespace identifiers
314 Returns:
315 The namespace that the given identifier is part of or None.
316 """
317 identifier = identifier.rsplit('.', 1)[0]
318 identifier_prefix = identifier + '.'
319 for namespaces in namespaces_list:
320 for namespace in namespaces:
321 if namespace == identifier or namespace.startswith(identifier_prefix):
322 return namespace
323 return None
324
325 def _IsPrivateIdentifier(self, identifier):
326 """Returns whether the given identifier is private."""
327 pieces = identifier.split('.')
328 for piece in pieces:
329 if piece.endswith('_'):
330 return True
331 return False
332
333 def IsFirstProvide(self, token):
334 """Returns whether token is the first provide token."""
335 return self._provide_tokens and token == self._provide_tokens[0]
336
337 def IsFirstRequire(self, token):
338 """Returns whether token is the first require token."""
339 return self._require_tokens and token == self._require_tokens[0]
340
341 def IsLastProvide(self, token):
342 """Returns whether token is the last provide token."""
343 return self._provide_tokens and token == self._provide_tokens[-1]
344
345 def IsLastRequire(self, token):
346 """Returns whether token is the last require token."""
347 return self._require_tokens and token == self._require_tokens[-1]
348
349 def ProcessToken(self, token, state_tracker):
350 """Processes the given token for dependency information.
351
352 Args:
353 token: The token to process.
354 state_tracker: The JavaScript state tracker.
355 """
356
357 # Note that this method is in the critical path for the linter and has been
358 # optimized for performance in the following ways:
359 # - Tokens are checked by type first to minimize the number of function
360 # calls necessary to determine if action needs to be taken for the token.
361 # - The most common tokens types are checked for first.
362 # - The number of function calls has been minimized (thus the length of this
363 # function.
364
365 if token.type == TokenType.IDENTIFIER:
366 # TODO(user): Consider saving the whole identifier in metadata.
367 whole_identifier_string = tokenutil.GetIdentifierForToken(token)
368 if whole_identifier_string is None:
369 # We only want to process the identifier one time. If the whole string
370 # identifier is None, that means this token was part of a multi-token
371 # identifier, but it was not the first token of the identifier.
372 return
373
374 # In the odd case that a goog.require is encountered inside a function,
375 # just ignore it (e.g. dynamic loading in test runners).
376 if token.string == 'goog.require' and not state_tracker.InFunction():
377 self._require_tokens.append(token)
378 namespace = tokenutil.GetStringAfterToken(token)
379 if namespace in self._required_namespaces:
380 self._duplicate_require_tokens.append(token)
381 else:
382 self._required_namespaces.append(namespace)
383
384 # If there is a suppression for the require, add a usage for it so it
385 # gets treated as a regular goog.require (i.e. still gets sorted).
386 if self._HasSuppression(state_tracker, 'extraRequire'):
387 self._suppressed_requires.append(namespace)
388 self._AddUsedNamespace(state_tracker, namespace, token)
389
390 elif token.string == 'goog.provide':
391 self._provide_tokens.append(token)
392 namespace = tokenutil.GetStringAfterToken(token)
393 if namespace in self._provided_namespaces:
394 self._duplicate_provide_tokens.append(token)
395 else:
396 self._provided_namespaces.append(namespace)
397
398 # If there is a suppression for the provide, add a creation for it so it
399 # gets treated as a regular goog.provide (i.e. still gets sorted).
400 if self._HasSuppression(state_tracker, 'extraProvide'):
401 self._AddCreatedNamespace(state_tracker, namespace, token.line_number)
402
403 elif token.string == 'goog.scope':
404 self._scopified_file = True
405
406 elif token.string == 'goog.setTestOnly':
407
408 # Since the message is optional, we don't want to scan to later lines.
409 for t in tokenutil.GetAllTokensInSameLine(token):
410 if t.type == TokenType.STRING_TEXT:
411 message = t.string
412
413 if re.match(r'^\w+(\.\w+)+$', message):
414 # This looks like a namespace. If it's a Closurized namespace,
415 # consider it created.
416 base_namespace = message.split('.', 1)[0]
417 if base_namespace in self._closurized_namespaces:
418 self._AddCreatedNamespace(state_tracker, message,
419 token.line_number)
420
421 break
422 else:
423 jsdoc = state_tracker.GetDocComment()
424 if token.metadata and token.metadata.aliased_symbol:
425 whole_identifier_string = token.metadata.aliased_symbol
426 elif (token.string == 'goog.module.get' and
427 not self._HasSuppression(state_tracker, 'extraRequire')):
428 # Cannot use _AddUsedNamespace as this is not an identifier, but
429 # already the entire namespace that's required.
430 namespace = tokenutil.GetStringAfterToken(token)
431 namespace = UsedNamespace(namespace, namespace, token,
432 alias_definition=False)
433 self._used_namespaces.append(namespace)
434 if jsdoc and jsdoc.HasFlag('typedef'):
435 self._AddCreatedNamespace(state_tracker, whole_identifier_string,
436 token.line_number,
437 namespace=self.GetClosurizedNamespace(
438 whole_identifier_string))
439 else:
440 is_alias_definition = (token.metadata and
441 token.metadata.is_alias_definition)
442 self._AddUsedNamespace(state_tracker, whole_identifier_string,
443 token, is_alias_definition)
444
445 elif token.type == TokenType.SIMPLE_LVALUE:
446 identifier = token.values['identifier']
447 start_token = tokenutil.GetIdentifierStart(token)
448 if start_token and start_token != token:
449 # Multi-line identifier being assigned. Get the whole identifier.
450 identifier = tokenutil.GetIdentifierForToken(start_token)
451 else:
452 start_token = token
453 # If an alias is defined on the start_token, use it instead.
454 if (start_token and
455 start_token.metadata and
456 start_token.metadata.aliased_symbol and
457 not start_token.metadata.is_alias_definition):
458 identifier = start_token.metadata.aliased_symbol
459
460 if identifier:
461 namespace = self.GetClosurizedNamespace(identifier)
462 if state_tracker.InFunction():
463 self._AddUsedNamespace(state_tracker, identifier, token)
464 elif namespace and namespace != 'goog':
465 self._AddCreatedNamespace(state_tracker, identifier,
466 token.line_number, namespace=namespace)
467
468 elif token.type == TokenType.DOC_FLAG:
469 flag = token.attached_object
470 flag_type = flag.flag_type
471 if flag and flag.HasType() and flag.jstype:
472 is_interface = state_tracker.GetDocComment().HasFlag('interface')
473 if flag_type == 'implements' or (flag_type == 'extends'
474 and is_interface):
475 identifier = flag.jstype.alias or flag.jstype.identifier
476 self._AddUsedNamespace(state_tracker, identifier, token)
477 # Since we process doctypes only for implements and extends, the
478 # type is a simple one and we don't need any iteration for subtypes.
479
480 def _AddCreatedNamespace(self, state_tracker, identifier, line_number,
481 namespace=None):
482 """Adds the namespace of an identifier to the list of created namespaces.
483
484 If the identifier is annotated with a 'missingProvide' suppression, it is
485 not added.
486
487 Args:
488 state_tracker: The JavaScriptStateTracker instance.
489 identifier: The identifier to add.
490 line_number: Line number where namespace is created.
491 namespace: The namespace of the identifier or None if the identifier is
492 also the namespace.
493 """
494 if not namespace:
495 namespace = identifier
496
497 if self._HasSuppression(state_tracker, 'missingProvide'):
498 return
499
500 self._created_namespaces.append([namespace, identifier, line_number])
501
502 def _AddUsedNamespace(self, state_tracker, identifier, token,
503 is_alias_definition=False):
504 """Adds the namespace of an identifier to the list of used namespaces.
505
506 If the identifier is annotated with a 'missingRequire' suppression, it is
507 not added.
508
509 Args:
510 state_tracker: The JavaScriptStateTracker instance.
511 identifier: An identifier which has been used.
512 token: The token in which the namespace is used.
513 is_alias_definition: If the used namespace is part of an alias_definition.
514 Aliased symbols need their parent namespace to be available, if it is
515 not yet required through another symbol, an error will be thrown.
516 """
517 if self._HasSuppression(state_tracker, 'missingRequire'):
518 return
519
520 identifier = self._GetUsedIdentifier(identifier)
521 namespace = self.GetClosurizedNamespace(identifier)
522 # b/5362203 If its a variable in scope then its not a required namespace.
523 if namespace and not state_tracker.IsVariableInScope(namespace):
524 namespace = UsedNamespace(namespace, identifier, token,
525 is_alias_definition)
526 self._used_namespaces.append(namespace)
527
528 def _HasSuppression(self, state_tracker, suppression):
529 jsdoc = state_tracker.GetDocComment()
530 return jsdoc and suppression in jsdoc.suppressions
531
532 def _GetUsedIdentifier(self, identifier):
533 """Strips apply/call/inherit calls from the identifier."""
534 for suffix in ('.apply', '.call', '.inherit'):
535 if identifier.endswith(suffix):
536 return identifier[:-len(suffix)]
537 return identifier
538
539 def GetClosurizedNamespace(self, identifier):
540 """Given an identifier, returns the namespace that identifier is from.
541
542 Args:
543 identifier: The identifier to extract a namespace from.
544
545 Returns:
546 The namespace the given identifier resides in, or None if one could not
547 be found.
548 """
549 if identifier.startswith('goog.global'):
550 # Ignore goog.global, since it is, by definition, global.
551 return None
552
553 parts = identifier.split('.')
554 for namespace in self._closurized_namespaces:
555 if not identifier.startswith(namespace + '.'):
556 continue
557
558 # The namespace for a class is the shortest prefix ending in a class
559 # name, which starts with a capital letter but is not a capitalized word.
560 #
561 # We ultimately do not want to allow requiring or providing of inner
562 # classes/enums. Instead, a file should provide only the top-level class
563 # and users should require only that.
564 namespace = []
565 for part in parts:
566 if part == 'prototype' or part.isupper():
567 return '.'.join(namespace)
568 namespace.append(part)
569 if part[0].isupper():
570 return '.'.join(namespace)
571
572 # At this point, we know there's no class or enum, so the namespace is
573 # just the identifier with the last part removed. With the exception of
574 # apply, inherits, and call, which should also be stripped.
575 if parts[-1] in ('apply', 'inherits', 'call'):
576 parts.pop()
577 parts.pop()
578
579 # If the last part ends with an underscore, it is a private variable,
580 # method, or enum. The namespace is whatever is before it.
581 if parts and parts[-1].endswith('_'):
582 parts.pop()
583
584 return '.'.join(parts)
585
586 return None
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698