Update //third_party/protobuf to version 3.

third_party/protobuf/objectivec/DevTools/pddm.py

+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2015 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""PDDM - Poor Developers' Debug-able Macros
+
+A simple markup that can be added in comments of source so they can then be
+expanded out into code. Most of this could be done with CPP macros, but then
+developers can't really step through them in the debugger, this way they are
+expanded to the same code, but you can debug them.
+
+Any file can be processed, but the syntax is designed around a C based compiler.
+Processed lines start with "//%".  There are three types of sections you can
+create: Text (left alone), Macro Definitions, and Macro Expansions.  There is
+no order required between definitions and expansions, all definitions are read
+before any expansions are processed (thus, if desired, definitions can be put
+at the end of the file to keep them out of the way of the code).
+
+Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines
+afterwards that start with "//%" are included in the definition.  Multiple
+macros can be defined in one block by just using a another "//%PDDM-DEFINE"
+line to start the next macro.  Optionally, a macro can be ended with
+"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that
+trailing blank lines are included in the macro.  You can also end a definition
+with an expansion.
+
+Macro Expansions are started by single lines containing
+"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another
+expansions.  All lines in-between are replaced by the result of the expansion.
+The first line of the expansion is always a blank like just for readability.
+
+Expansion itself is pretty simple, one macro can invoke another macro, but
+you cannot nest the invoke of a macro in another macro (i.e. - can't do
+"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo()
+within its expansion.
+
+When macros are expanded, the arg references can also add "$O" suffix to the
+name (i.e. - "NAME$O") to specify an option to be applied. The options are:
+
+    $S - Replace each character in the value with a space.
+    $l - Lowercase the first letter of the value.
+    $L - Lowercase the whole value.
+    $u - Uppercase the first letter of the value.
+    $U - Uppercase the whole value.
+
+Within a macro you can use ## to cause things to get joined together after
+expansion (i.e. - "a##b" within a macro will become "ab").
+
+Example:
+
+    int foo(MyEnum x) {
+    switch (x) {
+    //%PDDM-EXPAND case(Enum_Left, 1)
+    //%PDDM-EXPAND case(Enum_Center, 2)
+    //%PDDM-EXPAND case(Enum_Right, 3)
+    //%PDDM-EXPAND-END
+    }
+
+    //%PDDM-DEFINE case(_A, _B)
+    //%  case _A:
+    //%    return _B;
+
+  A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END
+  can be used to avoid adding extra blank lines/returns (or make it clear when
+  it is desired).
+
+  One macro can invoke another by simply using its name NAME(ARGS). You cannot
+  nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported).
+
+  Within a macro you can use ## to cause things to get joined together after
+  processing (i.e. - "a##b" within a macro will become "ab").
+
+
+"""
+
+import optparse
+import os
+import re
+import sys
+
+
+# Regex for macro definition.
+_MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)')
+# Regex for macro's argument definition.
+_MACRO_ARG_NAME_RE = re.compile(r'^\w+$')
+
+# Line inserted after each EXPAND.
+_GENERATED_CODE_LINE = (
+  '// This block of code is generated, do not edit it directly.'
+)
+
+
+def _MacroRefRe(macro_names):
+  # Takes in a list of macro names and makes a regex that will match invokes
+  # of those macros.
+  return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' %
+                    '|'.join(macro_names))
+
+def _MacroArgRefRe(macro_arg_names):
+  # Takes in a list of macro arg names and makes a regex that will match
+  # uses of those args.
+  return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' %
+                    '|'.join(macro_arg_names))
+
+
+class PDDMError(Exception):
+  """Error thrown by pddm."""
+  pass
+
+
+class MacroCollection(object):
+  """Hold a set of macros and can resolve/expand them."""
+
+  def __init__(self, a_file=None):
+    """Initializes the collection.
+
+    Args:
+      a_file: The file like stream to parse.
+
+    Raises:
+      PDDMError if there are any issues.
+    """
+    self._macros = dict()
+    if a_file:
+      self.ParseInput(a_file)
+
+  class MacroDefinition(object):
+    """Holds a macro definition."""
+
+    def __init__(self, name, arg_names):
+      self._name = name
+      self._args = tuple(arg_names)
+      self._body = ''
+      self._needNewLine = False
+
+    def AppendLine(self, line):
+      if self._needNewLine:
+        self._body += '\n'
+      self._body += line
+      self._needNewLine = not line.endswith('\n')
+
+    @property
+    def name(self):
+      return self._name
+
+    @property
+    def args(self):
+      return self._args
+
+    @property
+    def body(self):
+      return self._body
+
+  def ParseInput(self, a_file):
+    """Consumes input extracting definitions.
+
+    Args:
+      a_file: The file like stream to parse.
+
+    Raises:
+      PDDMError if there are any issues.
+    """
+    input_lines = a_file.read().splitlines()
+    self.ParseLines(input_lines)
+
+  def ParseLines(self, input_lines):
+    """Parses list of lines.
+
+    Args:
+      input_lines: A list of strings of input to parse (no newlines on the
+                   strings).
+
+    Raises:
+      PDDMError if there are any issues.
+    """
+    current_macro = None
+    for line in input_lines:
+      if line.startswith('PDDM-'):
+        directive = line.split(' ', 1)[0]
+        if directive == 'PDDM-DEFINE':
+          name, args = self._ParseDefineLine(line)
+          if self._macros.get(name):
+            raise PDDMError('Attempt to redefine macro: "%s"' % line)
+          current_macro = self.MacroDefinition(name, args)
+          self._macros[name] = current_macro
+          continue
+        if directive == 'PDDM-DEFINE-END':
+          if not current_macro:
+            raise PDDMError('Got DEFINE-END directive without an active macro:'
+                            ' "%s"' % line)
+          current_macro = None
+          continue
+        raise PDDMError('Hit a line with an unknown directive: "%s"' % line)
+
+      if current_macro:
+        current_macro.AppendLine(line)
+        continue
+
+      # Allow blank lines between macro definitions.
+      if line.strip() == '':
+        continue
+
+      raise PDDMError('Hit a line that wasn\'t a directive and no open macro'
+                      ' definition: "%s"' % line)
+
+  def _ParseDefineLine(self, input_line):
+    assert input_line.startswith('PDDM-DEFINE')
+    line = input_line[12:].strip()
+    match = _MACRO_RE.match(line)
+    # Must match full line
+    if match is None or match.group(0) != line:
+      raise PDDMError('Failed to parse macro definition: "%s"' % input_line)
+    name = match.group('name')
+    args_str = match.group('args').strip()
+    args = []
+    if args_str:
+      for part in args_str.split(','):
+        arg = part.strip()
+        if arg == '':
+          raise PDDMError('Empty arg name in macro definition: "%s"'
+                          % input_line)
+        if not _MACRO_ARG_NAME_RE.match(arg):
+          raise PDDMError('Invalid arg name "%s" in macro definition: "%s"'
+                          % (arg, input_line))
+        if arg in args:
+          raise PDDMError('Arg name "%s" used more than once in macro'
+                          ' definition: "%s"' % (arg, input_line))
+        args.append(arg)
+    return (name, tuple(args))
+
+  def Expand(self, macro_ref_str):
+    """Expands the macro reference.
+
+    Args:
+      macro_ref_str: String of a macro reference (i.e. foo(a, b)).
+
+    Returns:
+      The text from the expansion.
+
+    Raises:
+      PDDMError if there are any issues.
+    """
+    match = _MACRO_RE.match(macro_ref_str)
+    if match is None or match.group(0) != macro_ref_str:
+      raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str)
+    if match.group('name') not in self._macros:
+      raise PDDMError('No macro named "%s".' % match.group('name'))
+    return self._Expand(match, [], macro_ref_str)
+
+  def _FormatStack(self, macro_ref_stack):
+    result = ''
+    for _, macro_ref in reversed(macro_ref_stack):
+      result += '\n...while expanding "%s".' % macro_ref
+    return result
+
+  def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None):
+    if macro_ref_str is None:
+      macro_ref_str = macro_ref_match.group('macro_ref')
+    name = macro_ref_match.group('name')
+    for prev_name, prev_macro_ref in macro_stack:
+      if name == prev_name:
+        raise PDDMError('Found macro recusion, invoking "%s":%s' %
+                        (macro_ref_str, self._FormatStack(macro_stack)))
+    macro = self._macros[name]
+    args_str = macro_ref_match.group('args').strip()
+    args = []
+    if args_str or len(macro.args):
+      args = [x.strip() for x in args_str.split(',')]
+    if len(args) != len(macro.args):
+      raise PDDMError('Expected %d args, got: "%s".%s' %
+                      (len(macro.args), macro_ref_str,
+                       self._FormatStack(macro_stack)))
+    # Replace args usages.
+    result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack)
+    # Expand any macro invokes.
+    new_macro_stack = macro_stack + [(name, macro_ref_str)]
+    while True:
+      eval_result = self._EvalMacrosRefs(result, new_macro_stack)
+      # Consume all ## directives to glue things together.
+      eval_result = eval_result.replace('##', '')
+      if eval_result == result:
+        break
+      result = eval_result
+    return result
+
+  def _ReplaceArgValues(self,
+                        macro, arg_values, macro_ref_to_report, macro_stack):
+    if len(arg_values) == 0:
+      # Nothing to do
+      return macro.body
+    assert len(arg_values) == len(macro.args)
+    args = dict(zip(macro.args, arg_values))
+    def _lookupArg(match):
+      val = args[match.group('name')]
+      opt = match.group('option')
+      if opt:
+        if opt == 'S': # Spaces for the length
+          return ' ' * len(val)
+        elif opt == 'l': # Lowercase first character
+          if val:
+            return val[0].lower() + val[1:]
+          else:
+            return val
+        elif opt == 'L': # All Lowercase
+          return val.lower()
+        elif opt == 'u': # Uppercase first character
+          if val:
+            return val[0].upper() + val[1:]
+          else:
+            return val
+        elif opt == 'U': # All Uppercase
+          return val.upper()
+        else:
+          raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s'
+                          % (match.group('name'), match.group('option'),
+                             macro_ref_to_report,
+                             self._FormatStack(macro_stack)))
+      return val
+    # Let the regex do the work!
+    macro_arg_ref_re = _MacroArgRefRe(macro.args)
+    return macro_arg_ref_re.sub(_lookupArg, macro.body)
+
+  def _EvalMacrosRefs(self, text, macro_stack):
+    macro_ref_re = _MacroRefRe(self._macros.keys())
+    def _resolveMacro(match):
+      return self._Expand(match, macro_stack)
+    return macro_ref_re.sub(_resolveMacro, text)
+
+
+class SourceFile(object):
+  """Represents a source file with PDDM directives in it."""
+
+  def __init__(self, a_file, import_resolver=None):
+    """Initializes the file reading in the file.
+
+    Args:
+      a_file: The file to read in.
+      import_resolver: a function that given a path will return a stream for
+        the contents.
+
+    Raises:
+      PDDMError if there are any issues.
+    """
+    self._sections = []
+    self._original_content = a_file.read()
+    self._import_resolver = import_resolver
+    self._processed_content = None
+
+  class SectionBase(object):
+
+    def __init__(self, first_line_num):
+      self._lines = []
+      self._first_line_num = first_line_num
+
+    def TryAppend(self, line, line_num):
+      """Try appending a line.
+
+      Args:
+        line: The line to append.
+        line_num: The number of the line.
+
+      Returns:
+        A tuple of (SUCCESS, CAN_ADD_MORE).  If SUCCESS if False, the line
+        wasn't append.  If SUCCESS is True, then CAN_ADD_MORE is True/False to
+        indicate if more lines can be added after this one.
+      """
+      assert False, "sublcass should have overridden"
+      return (False, False)
+
+    def HitEOF(self):
+      """Called when the EOF was reached for for a given section."""
+      pass
+
+    def BindMacroCollection(self, macro_collection):
+      """Binds the chunk to a macro collection.
+
+      Args:
+        macro_collection: The collection to bind too.
+      """
+      pass
+
+    def Append(self, line):
+      self._lines.append(line)
+
+    @property
+    def lines(self):
+      return self._lines
+
+    @property
+    def num_lines_captured(self):
+      return len(self._lines)
+
+    @property
+    def first_line_num(self):
+      return self._first_line_num
+
+    @property
+    def first_line(self):
+      if not self._lines:
+        return ''
+      return self._lines[0]
+
+    @property
+    def text(self):
+      return '\n'.join(self.lines) + '\n'
+
+  class TextSection(SectionBase):
+    """Text section that is echoed out as is."""
+
+    def TryAppend(self, line, line_num):
+      if line.startswith('//%PDDM'):
+        return (False, False)
+      self.Append(line)
+      return (True, True)
+
+  class ExpansionSection(SectionBase):
+    """Section that is the result of an macro expansion."""
+
+    def __init__(self, first_line_num):
+      SourceFile.SectionBase.__init__(self, first_line_num)
+      self._macro_collection = None
+
+    def TryAppend(self, line, line_num):
+      if line.startswith('//%PDDM'):
+        directive = line.split(' ', 1)[0]
+        if directive == '//%PDDM-EXPAND':
+          self.Append(line)
+          return (True, True)
+        if directive == '//%PDDM-EXPAND-END':
+          assert self.num_lines_captured > 0
+          return (True, False)
+        raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' %
+                        (directive, line_num, self.first_line))
+      # Eat other lines.
+      return (True, True)
+
+    def HitEOF(self):
+      raise PDDMError('Hit the end of the file while in "%s".' %
+                      self.first_line)
+
+    def BindMacroCollection(self, macro_collection):
+      self._macro_collection = macro_collection
+
+    @property
+    def lines(self):
+      captured_lines = SourceFile.SectionBase.lines.fget(self)
+      directive_len = len('//%PDDM-EXPAND')
+      result = []
+      for line in captured_lines:
+        result.append(line)
+        if self._macro_collection:
+          # Always add a blank line, seems to read better. (If need be, add an
+          # option to the EXPAND to indicate if this should be done.)
+          result.extend([_GENERATED_CODE_LINE, ''])
+          macro = line[directive_len:].strip()
+          try:
+            expand_result = self._macro_collection.Expand(macro)
+            # Since expansions are line oriented, strip trailing whitespace
+            # from the lines.
+            lines = [x.rstrip() for x in expand_result.split('\n')]
+            result.append('\n'.join(lines))
+          except PDDMError as e:
+            raise PDDMError('%s\n...while expanding "%s" from the section'
+                            ' that started:\n   Line %d: %s' %
+                            (e.message, macro,
+                             self.first_line_num, self.first_line))
+
+      # Add the ending marker.
+      if len(captured_lines) == 1:
+        result.append('//%%PDDM-EXPAND-END %s' %
+                       captured_lines[0][directive_len:].strip())
+      else:
+        result.append('//%%PDDM-EXPAND-END (%s expansions)' % len(captured_lines))
+
+      return result
+
+  class DefinitionSection(SectionBase):
+    """Section containing macro definitions"""
+
+    def TryAppend(self, line, line_num):
+      if not line.startswith('//%'):
+        return (False, False)
+      if line.startswith('//%PDDM'):
+        directive = line.split(' ', 1)[0]
+        if directive == "//%PDDM-EXPAND":
+          return False, False
+        if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'):
+          raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' %
+                          (directive, line_num, self.first_line))
+      self.Append(line)
+      return (True, True)
+
+    def BindMacroCollection(self, macro_collection):
+      if macro_collection:
+        try:
+          # Parse the lines after stripping the prefix.
+          macro_collection.ParseLines([x[3:] for x in self.lines])
+        except PDDMError as e:
+          raise PDDMError('%s\n...while parsing section that started:\n'
+                          '  Line %d: %s' %
+                          (e.message, self.first_line_num, self.first_line))
+
+  class ImportDefinesSection(SectionBase):
+    """Section containing an import of PDDM-DEFINES from an external file."""
+
+    def __init__(self, first_line_num, import_resolver):
+      SourceFile.SectionBase.__init__(self, first_line_num)
+      self._import_resolver = import_resolver
+
+    def TryAppend(self, line, line_num):
+      if not line.startswith('//%PDDM-IMPORT-DEFINES '):
+        return (False, False)
+      assert self.num_lines_captured == 0
+      self.Append(line)
+      return (True, False)
+
+    def BindMacroCollection(self, macro_colletion):
+      if not macro_colletion:
+        return
+      if self._import_resolver is None:
+        raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):'
+                        ' "%s".' % (self.first_line_num, self.first_line))
+      import_name = self.first_line.split(' ', 1)[1].strip()
+      imported_file = self._import_resolver(import_name)
+      if imported_file is None:
+        raise PDDMError('Resolver failed to find "%s" (line %d):'
+                        ' "%s".' %
+                        (import_name, self.first_line_num, self.first_line))
+      try:
+        imported_src_file = SourceFile(imported_file, self._import_resolver)
+        imported_src_file._ParseFile()
+        for section in imported_src_file._sections:
+          section.BindMacroCollection(macro_colletion)
+      except PDDMError as e:
+        raise PDDMError('%s\n...while importing defines:\n'
+                        '  Line %d: %s' %
+                        (e.message, self.first_line_num, self.first_line))
+
+  def _ParseFile(self):
+    self._sections = []
+    lines = self._original_content.splitlines()
+    cur_section = None
+    for line_num, line in enumerate(lines, 1):
+      if not cur_section:
+        cur_section = self._MakeSection(line, line_num)
+      was_added, accept_more = cur_section.TryAppend(line, line_num)
+      if not was_added:
+        cur_section = self._MakeSection(line, line_num)
+        was_added, accept_more = cur_section.TryAppend(line, line_num)
+        assert was_added
+      if not accept_more:
+        cur_section = None
+
+    if cur_section:
+      cur_section.HitEOF()
+
+  def _MakeSection(self, line, line_num):
+    if not line.startswith('//%PDDM'):
+      section = self.TextSection(line_num)
+    else:
+      directive = line.split(' ', 1)[0]
+      if directive == '//%PDDM-EXPAND':
+        section = self.ExpansionSection(line_num)
+      elif directive == '//%PDDM-DEFINE':
+        section = self.DefinitionSection(line_num)
+      elif directive == '//%PDDM-IMPORT-DEFINES':
+        section = self.ImportDefinesSection(line_num, self._import_resolver)
+      else:
+        raise PDDMError('Unexpected line %d: "%s".' % (line_num, line))
+    self._sections.append(section)
+    return section
+
+  def ProcessContent(self, strip_expansion=False):
+    """Processes the file contents."""
+    self._ParseFile()
+    if strip_expansion:
+      # Without a collection the expansions become blank, removing them.
+      collection = None
+    else:
+      collection = MacroCollection()
+    for section in self._sections:
+      section.BindMacroCollection(collection)
+    result = ''
+    for section in self._sections:
+      result += section.text
+    self._processed_content = result
+
+  @property
+  def original_content(self):
+    return self._original_content
+
+  @property
+  def processed_content(self):
+    return self._processed_content
+
+
+def main(args):
+  usage = '%prog [OPTIONS] PATH ...'
+  description = (
+      'Processes PDDM directives in the the given paths and write them back'
+      ' out.'
+  )
+  parser = optparse.OptionParser(usage=usage, description=description)
+  parser.add_option('--dry-run',
+                    default=False, action='store_true',
+                    help='Don\'t write back to the file(s), just report if the'
+                    ' contents needs an update and exit with a value of 1.')
+  parser.add_option('--verbose',
+                    default=False, action='store_true',
+                    help='Reports is a file is already current.')
+  parser.add_option('--collapse',
+                    default=False, action='store_true',
+                    help='Removes all the generated code.')
+  opts, extra_args = parser.parse_args(args)
+
+  if not extra_args:
+    parser.error('Need atleast one file to process')
+
+  result = 0
+  for a_path in extra_args:
+    if not os.path.exists(a_path):
+      sys.stderr.write('ERROR: File not found: %s\n' % a_path)
+      return 100
+
+    def _ImportResolver(name):
+      # resolve based on the file being read.
+      a_dir = os.path.dirname(a_path)
+      import_path = os.path.join(a_dir, name)
+      if not os.path.exists(import_path):
+        return None
+      return open(import_path, 'r')
+
+    with open(a_path, 'r') as f:
+      src_file = SourceFile(f, _ImportResolver)
+
+    try:
+      src_file.ProcessContent(strip_expansion=opts.collapse)
+    except PDDMError as e:
+      sys.stderr.write('ERROR: %s\n...While processing "%s"\n' %
+                       (e.message, a_path))
+      return 101
+
+    if src_file.processed_content != src_file.original_content:
+      if not opts.dry_run:
+        print 'Updating for "%s".' % a_path
+        with open(a_path, 'w') as f:
+          f.write(src_file.processed_content)
+      else:
+        # Special result to indicate things need updating.
+        print 'Update needed for "%s".' % a_path
+        result = 1
+    elif opts.verbose:
+      print 'No update for "%s".' % a_path
+
+  return result
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))