Update protobuf to r428, part 1.

third_party/protobuf/src/google/protobuf/compiler/parser.cc

Index: third_party/protobuf/src/google/protobuf/compiler/parser.cc
===================================================================
--- third_party/protobuf/src/google/protobuf/compiler/parser.cc	(revision 216642)
+++ third_party/protobuf/src/google/protobuf/compiler/parser.cc	(working copy)
@@ -174,6 +174,20 @@
   }
 }
 
+bool Parser::ConsumeSignedInteger(int* output, const char* error) {
+  bool is_negative = false;
+  uint64 max_value = kint32max;
+  if (TryConsume("-")) {
+    is_negative = true;
+    max_value += 1;
+  }
+  uint64 value = 0;
+  DO(ConsumeInteger64(max_value, &value, error));
+  if (is_negative) value *= -1;
+  *output = value;
+  return true;
+}
+
 bool Parser::ConsumeInteger64(uint64 max_value, uint64* output,
                               const char* error) {
   if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
@@ -237,6 +251,35 @@
   }
 }
 
+bool Parser::TryConsumeEndOfDeclaration(const char* text,
+                                        const LocationRecorder* location) {
+  if (LookingAt(text)) {
+    string leading, trailing;
+    input_->NextWithComments(&trailing, NULL, &leading);
+
+    // Save the leading comments for next time, and recall the leading comments
+    // from last time.
+    leading.swap(upcoming_doc_comments_);
+
+    if (location != NULL) {
+      location->AttachComments(&leading, &trailing);
+    }
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool Parser::ConsumeEndOfDeclaration(const char* text,
+                                     const LocationRecorder* location) {
+  if (TryConsumeEndOfDeclaration(text, location)) {
+    return true;
+  } else {
+    AddError("Expected \"" + string(text) + "\".");
+    return false;
+  }
+}
+
 // -------------------------------------------------------------------
 
 void Parser::AddError(int line, int column, const string& error) {
@@ -315,6 +358,19 @@
   }
 }
 
+void Parser::LocationRecorder::AttachComments(
+    string* leading, string* trailing) const {
+  GOOGLE_CHECK(!location_->has_leading_comments());
+  GOOGLE_CHECK(!location_->has_trailing_comments());
+
+  if (!leading->empty()) {
+    location_->mutable_leading_comments()->swap(*leading);
+  }
+  if (!trailing->empty()) {
+    location_->mutable_trailing_comments()->swap(*trailing);
+  }
+}
+
 // -------------------------------------------------------------------
 
 void Parser::SkipStatement() {
@@ -322,7 +378,7 @@
     if (AtEnd()) {
       return;
     } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
-      if (TryConsume(";")) {
+      if (TryConsumeEndOfDeclaration(";", NULL)) {
         return;
       } else if (TryConsume("{")) {
         SkipRestOfBlock();
@@ -340,7 +396,7 @@
     if (AtEnd()) {
       return;
     } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
-      if (TryConsume("}")) {
+      if (TryConsumeEndOfDeclaration("}", NULL)) {
         return;
       } else if (TryConsume("{")) {
         SkipRestOfBlock();
@@ -366,7 +422,7 @@
 
   if (LookingAtType(io::Tokenizer::TYPE_START)) {
     // Advance to first token.
-    input_->Next();
+    input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_);
   }
 
   {
@@ -393,7 +449,7 @@
 
         if (LookingAt("}")) {
           AddError("Unmatched \"}\".");
-          input_->Next();
+          input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_);
         }
       }
     }
@@ -411,7 +467,7 @@
   io::Tokenizer::Token syntax_token = input_->current();
   string syntax;
   DO(ConsumeString(&syntax, "Expected syntax identifier."));
-  DO(Consume(";"));
+  DO(ConsumeEndOfDeclaration(";", NULL));
 
   syntax_identifier_ = syntax;
 
@@ -427,7 +483,7 @@
 
 bool Parser::ParseTopLevelStatement(FileDescriptorProto* file,
                                     const LocationRecorder& root_location) {
-  if (TryConsume(";")) {
+  if (TryConsumeEndOfDeclaration(";", NULL)) {
     // empty statement; ignore
     return true;
   } else if (LookingAt("message")) {
@@ -451,14 +507,16 @@
                        FileDescriptorProto::kMessageTypeFieldNumber,
                        location);
   } else if (LookingAt("import")) {
-    int index = file->dependency_size();
-    return ParseImport(file->add_dependency(), root_location, index);
+    return ParseImport(file->mutable_dependency(),
+                       file->mutable_public_dependency(),
+                       file->mutable_weak_dependency(),
+                       root_location);
   } else if (LookingAt("package")) {
     return ParsePackage(file, root_location);
   } else if (LookingAt("option")) {
     LocationRecorder location(root_location,
         FileDescriptorProto::kOptionsFieldNumber);
-    return ParseOption(file->mutable_options(), location);
+    return ParseOption(file->mutable_options(), location, OPTION_STATEMENT);
   } else {
     AddError("Expected top-level statement (e.g. \"message\").");
     return false;
@@ -482,11 +540,45 @@
   return true;
 }
 
+namespace {
+
+const int kMaxExtensionRangeSentinel = -1;
+
+bool IsMessageSetWireFormatMessage(const DescriptorProto& message) {
+  const MessageOptions& options = message.options();
+  for (int i = 0; i < options.uninterpreted_option_size(); ++i) {
+    const UninterpretedOption& uninterpreted = options.uninterpreted_option(i);
+    if (uninterpreted.name_size() == 1 &&
+        uninterpreted.name(0).name_part() == "message_set_wire_format" &&
+        uninterpreted.identifier_value() == "true") {
+      return true;
+    }
+  }
+  return false;
+}
+
+// Modifies any extension ranges that specified 'max' as the end of the
+// extension range, and sets them to the type-specific maximum. The actual max
+// tag number can only be determined after all options have been parsed.
+void AdjustExtensionRangesWithMaxEndNumber(DescriptorProto* message) {
+  const bool is_message_set = IsMessageSetWireFormatMessage(*message);
+  const int max_extension_number = is_message_set ?
+      kint32max :
+      FieldDescriptor::kMaxNumber + 1;
+  for (int i = 0; i < message->extension_range_size(); ++i) {
+    if (message->extension_range(i).end() == kMaxExtensionRangeSentinel) {
+      message->mutable_extension_range(i)->set_end(max_extension_number);
+    }
+  }
+}
+
+}  // namespace
+
 bool Parser::ParseMessageBlock(DescriptorProto* message,
                                const LocationRecorder& message_location) {
-  DO(Consume("{"));
+  DO(ConsumeEndOfDeclaration("{", &message_location));
 
-  while (!TryConsume("}")) {
+  while (!TryConsumeEndOfDeclaration("}", NULL)) {
     if (AtEnd()) {
       AddError("Reached end of input in message definition (missing '}').");
       return false;
@@ -499,12 +591,15 @@
     }
   }
 
+  if (message->extension_range_size() > 0) {
+    AdjustExtensionRangesWithMaxEndNumber(message);
+  }
   return true;
 }
 
 bool Parser::ParseMessageStatement(DescriptorProto* message,
                                    const LocationRecorder& message_location) {
-  if (TryConsume(";")) {
+  if (TryConsumeEndOfDeclaration(";", NULL)) {
     // empty statement; ignore
     return true;
   } else if (LookingAt("message")) {
@@ -532,7 +627,7 @@
   } else if (LookingAt("option")) {
     LocationRecorder location(message_location,
                               DescriptorProto::kOptionsFieldNumber);
-    return ParseOption(message->mutable_options(), location);
+    return ParseOption(message->mutable_options(), location, OPTION_STATEMENT);
   } else {
     LocationRecorder location(message_location,
                               DescriptorProto::kFieldFieldNumber,
@@ -647,7 +742,7 @@
       return false;
     }
   } else {
-    DO(Consume(";"));
+    DO(ConsumeEndOfDeclaration(";", &field_location));
   }
 
   return true;
@@ -669,7 +764,7 @@
       // the default value is not actually an option.
       DO(ParseDefaultAssignment(field, field_location));
     } else {
-      DO(ParseOptionAssignment(field->mutable_options(), location));
+      DO(ParseOption(field->mutable_options(), location, OPTION_ASSIGNMENT));
     }
   } while (TryConsume(","));
 
@@ -835,6 +930,8 @@
 
 bool Parser::ParseUninterpretedBlock(string* value) {
   // Note that enclosing braces are not added to *value.
+  // We do NOT use ConsumeEndOfStatement for this brace because it's delimiting
+  // an expression, not a block of statements.
   DO(Consume("{"));
   int brace_depth = 1;
   while (!AtEnd()) {
@@ -858,8 +955,9 @@
 
 // We don't interpret the option here. Instead we store it in an
 // UninterpretedOption, to be interpreted later.
-bool Parser::ParseOptionAssignment(Message* options,
-                                   const LocationRecorder& options_location) {
+bool Parser::ParseOption(Message* options,
+                         const LocationRecorder& options_location,
+                         OptionStyle style) {
   // Create an entry in the uninterpreted_option field.
   const FieldDescriptor* uninterpreted_option_field = options->GetDescriptor()->
       FindFieldByName("uninterpreted_option");
@@ -872,6 +970,10 @@
       options_location, uninterpreted_option_field->number(),
       reflection->FieldSize(*options, uninterpreted_option_field));
 
+  if (style == OPTION_STATEMENT) {
+    DO(Consume("option"));
+  }
+
   UninterpretedOption* uninterpreted_option = down_cast<UninterpretedOption*>(
       options->GetReflection()->AddMessage(options,
                                            uninterpreted_option_field));
@@ -899,84 +1001,93 @@
 
   DO(Consume("="));
 
-  LocationRecorder value_location(location);
-  value_location.RecordLegacyLocation(
-      uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_VALUE);
+  {
+    LocationRecorder value_location(location);
+    value_location.RecordLegacyLocation(
+        uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_VALUE);
 
-  // All values are a single token, except for negative numbers, which consist
-  // of a single '-' symbol, followed by a positive number.
-  bool is_negative = TryConsume("-");
+    // All values are a single token, except for negative numbers, which consist
+    // of a single '-' symbol, followed by a positive number.
+    bool is_negative = TryConsume("-");
 
-  switch (input_->current().type) {
-    case io::Tokenizer::TYPE_START:
-      GOOGLE_LOG(FATAL) << "Trying to read value before any tokens have been read.";
-      return false;
+    switch (input_->current().type) {
+      case io::Tokenizer::TYPE_START:
+        GOOGLE_LOG(FATAL) << "Trying to read value before any tokens have been read.";
+        return false;
 
-    case io::Tokenizer::TYPE_END:
-      AddError("Unexpected end of stream while parsing option value.");
-      return false;
-
-    case io::Tokenizer::TYPE_IDENTIFIER: {
-      value_location.AddPath(UninterpretedOption::kIdentifierValueFieldNumber);
-      if (is_negative) {
-        AddError("Invalid '-' symbol before identifier.");
+      case io::Tokenizer::TYPE_END:
+        AddError("Unexpected end of stream while parsing option value.");
         return false;
-      }
-      string value;
-      DO(ConsumeIdentifier(&value, "Expected identifier."));
-      uninterpreted_option->set_identifier_value(value);
-      break;
-    }
 
-    case io::Tokenizer::TYPE_INTEGER: {
-      uint64 value;
-      uint64 max_value =
-          is_negative ? static_cast<uint64>(kint64max) + 1 : kuint64max;
-      DO(ConsumeInteger64(max_value, &value, "Expected integer."));
-      if (is_negative) {
+      case io::Tokenizer::TYPE_IDENTIFIER: {
         value_location.AddPath(
-            UninterpretedOption::kNegativeIntValueFieldNumber);
-        uninterpreted_option->set_negative_int_value(-static_cast<int64>(value));
-      } else {
-        value_location.AddPath(
-            UninterpretedOption::kPositiveIntValueFieldNumber);
-        uninterpreted_option->set_positive_int_value(value);
+            UninterpretedOption::kIdentifierValueFieldNumber);
+        if (is_negative) {
+          AddError("Invalid '-' symbol before identifier.");
+          return false;
+        }
+        string value;
+        DO(ConsumeIdentifier(&value, "Expected identifier."));
+        uninterpreted_option->set_identifier_value(value);
+        break;
       }
-      break;
-    }
 
-    case io::Tokenizer::TYPE_FLOAT: {
-      value_location.AddPath(UninterpretedOption::kDoubleValueFieldNumber);
-      double value;
-      DO(ConsumeNumber(&value, "Expected number."));
-      uninterpreted_option->set_double_value(is_negative ? -value : value);
-      break;
-    }
+      case io::Tokenizer::TYPE_INTEGER: {
+        uint64 value;
+        uint64 max_value =
+            is_negative ? static_cast<uint64>(kint64max) + 1 : kuint64max;
+        DO(ConsumeInteger64(max_value, &value, "Expected integer."));
+        if (is_negative) {
+          value_location.AddPath(
+              UninterpretedOption::kNegativeIntValueFieldNumber);
+          uninterpreted_option->set_negative_int_value(
+              -static_cast<int64>(value));
+        } else {
+          value_location.AddPath(
+              UninterpretedOption::kPositiveIntValueFieldNumber);
+          uninterpreted_option->set_positive_int_value(value);
+        }
+        break;
+      }
 
-    case io::Tokenizer::TYPE_STRING: {
-      value_location.AddPath(UninterpretedOption::kStringValueFieldNumber);
-      if (is_negative) {
-        AddError("Invalid '-' symbol before string.");
-        return false;
+      case io::Tokenizer::TYPE_FLOAT: {
+        value_location.AddPath(UninterpretedOption::kDoubleValueFieldNumber);
+        double value;
+        DO(ConsumeNumber(&value, "Expected number."));
+        uninterpreted_option->set_double_value(is_negative ? -value : value);
+        break;
       }
-      string value;
-      DO(ConsumeString(&value, "Expected string."));
-      uninterpreted_option->set_string_value(value);
-      break;
-    }
 
-    case io::Tokenizer::TYPE_SYMBOL:
-      if (LookingAt("{")) {
-        value_location.AddPath(UninterpretedOption::kAggregateValueFieldNumber);
-        DO(ParseUninterpretedBlock(
-            uninterpreted_option->mutable_aggregate_value()));
-      } else {
-        AddError("Expected option value.");
-        return false;
+      case io::Tokenizer::TYPE_STRING: {
+        value_location.AddPath(UninterpretedOption::kStringValueFieldNumber);
+        if (is_negative) {
+          AddError("Invalid '-' symbol before string.");
+          return false;
+        }
+        string value;
+        DO(ConsumeString(&value, "Expected string."));
+        uninterpreted_option->set_string_value(value);
+        break;
       }
-      break;
+
+      case io::Tokenizer::TYPE_SYMBOL:
+        if (LookingAt("{")) {
+          value_location.AddPath(
+              UninterpretedOption::kAggregateValueFieldNumber);
+          DO(ParseUninterpretedBlock(
+              uninterpreted_option->mutable_aggregate_value()));
+        } else {
+          AddError("Expected option value.");
+          return false;
+        }
+        break;
+    }
   }
 
+  if (style == OPTION_STATEMENT) {
+    DO(ConsumeEndOfDeclaration(";", &location));
+  }
+
   return true;
 }
 
@@ -1008,7 +1119,10 @@
       LocationRecorder end_location(
           location, DescriptorProto::ExtensionRange::kEndFieldNumber);
       if (TryConsume("max")) {
-        end = FieldDescriptor::kMaxNumber;
+        // Set to the sentinel value - 1 since we increment the value below.
+        // The actual value of the end of the range should be set with
+        // AdjustExtensionRangesWithMaxEndNumber.
+        end = kMaxExtensionRangeSentinel - 1;
       } else {
         DO(ConsumeInteger(&end, "Expected integer."));
       }
@@ -1028,7 +1142,7 @@
     range->set_end(end);
   } while (TryConsume(","));
 
-  DO(Consume(";"));
+  DO(ConsumeEndOfDeclaration(";", &extensions_location));
   return true;
 }
 
@@ -1046,7 +1160,7 @@
   io::Tokenizer::Token extendee_end = input_->previous();
 
   // Parse the block.
-  DO(Consume("{"));
+  DO(ConsumeEndOfDeclaration("{", &extend_location));
 
   bool is_first = true;
 
@@ -1083,7 +1197,7 @@
       // other statements.
       SkipStatement();
     }
-  } while(!TryConsume("}"));
+  } while (!TryConsumeEndOfDeclaration("}", NULL));
 
   return true;
 }
@@ -1109,9 +1223,9 @@
 
 bool Parser::ParseEnumBlock(EnumDescriptorProto* enum_type,
                             const LocationRecorder& enum_location) {
-  DO(Consume("{"));
+  DO(ConsumeEndOfDeclaration("{", &enum_location));
 
-  while (!TryConsume("}")) {
+  while (!TryConsumeEndOfDeclaration("}", NULL)) {
     if (AtEnd()) {
       AddError("Reached end of input in enum definition (missing '}').");
       return false;
@@ -1129,13 +1243,14 @@
 
 bool Parser::ParseEnumStatement(EnumDescriptorProto* enum_type,
                                 const LocationRecorder& enum_location) {
-  if (TryConsume(";")) {
+  if (TryConsumeEndOfDeclaration(";", NULL)) {
     // empty statement; ignore
     return true;
   } else if (LookingAt("option")) {
     LocationRecorder location(enum_location,
                               EnumDescriptorProto::kOptionsFieldNumber);
-    return ParseOption(enum_type->mutable_options(), location);
+    return ParseOption(enum_type->mutable_options(), location,
+                       OPTION_STATEMENT);
   } else {
     LocationRecorder location(enum_location,
         EnumDescriptorProto::kValueFieldNumber, enum_type->value_size());
@@ -1164,16 +1279,14 @@
     location.RecordLegacyLocation(
         enum_value, DescriptorPool::ErrorCollector::NUMBER);
 
-    bool is_negative = TryConsume("-");
     int number;
-    DO(ConsumeInteger(&number, "Expected integer."));
-    if (is_negative) number *= -1;
+    DO(ConsumeSignedInteger(&number, "Expected integer."));
     enum_value->set_number(number);
   }
 
   DO(ParseEnumConstantOptions(enum_value, enum_value_location));
 
-  DO(Consume(";"));
+  DO(ConsumeEndOfDeclaration(";", &enum_value_location));
 
   return true;
 }
@@ -1189,7 +1302,7 @@
   DO(Consume("["));
 
   do {
-    DO(ParseOptionAssignment(value->mutable_options(), location));
+    DO(ParseOption(value->mutable_options(), location, OPTION_ASSIGNMENT));
   } while (TryConsume(","));
 
   DO(Consume("]"));
@@ -1217,9 +1330,9 @@
 
 bool Parser::ParseServiceBlock(ServiceDescriptorProto* service,
                                const LocationRecorder& service_location) {
-  DO(Consume("{"));
+  DO(ConsumeEndOfDeclaration("{", &service_location));
 
-  while (!TryConsume("}")) {
+  while (!TryConsumeEndOfDeclaration("}", NULL)) {
     if (AtEnd()) {
       AddError("Reached end of input in service definition (missing '}').");
       return false;
@@ -1237,13 +1350,13 @@
 
 bool Parser::ParseServiceStatement(ServiceDescriptorProto* service,
                                    const LocationRecorder& service_location) {
-  if (TryConsume(";")) {
+  if (TryConsumeEndOfDeclaration(";", NULL)) {
     // empty statement; ignore
     return true;
   } else if (LookingAt("option")) {
     LocationRecorder location(
         service_location, ServiceDescriptorProto::kOptionsFieldNumber);
-    return ParseOption(service->mutable_options(), location);
+    return ParseOption(service->mutable_options(), location, OPTION_STATEMENT);
   } else {
     LocationRecorder location(service_location,
         ServiceDescriptorProto::kMethodFieldNumber, service->method_size());
@@ -1286,28 +1399,41 @@
   }
   DO(Consume(")"));
 
-  if (TryConsume("{")) {
+  if (LookingAt("{")) {
     // Options!
-    while (!TryConsume("}")) {
-      if (AtEnd()) {
-        AddError("Reached end of input in method options (missing '}').");
-        return false;
-      }
+    DO(ParseOptions(method_location,
+                    MethodDescriptorProto::kOptionsFieldNumber,
+                    method->mutable_options()));
+  } else {
+    DO(ConsumeEndOfDeclaration(";", &method_location));
+  }
 
-      if (TryConsume(";")) {
-        // empty statement; ignore
-      } else {
-        LocationRecorder location(method_location,
-                                  MethodDescriptorProto::kOptionsFieldNumber);
-        if (!ParseOption(method->mutable_options(), location)) {
-          // This statement failed to parse.  Skip it, but keep looping to
-          // parse other statements.
-          SkipStatement();
-        }
+  return true;
+}
+
+
+bool Parser::ParseOptions(const LocationRecorder& parent_location,
+                          const int optionsFieldNumber,
+                          Message* mutable_options) {
+  // Options!
+  ConsumeEndOfDeclaration("{", &parent_location);
+  while (!TryConsumeEndOfDeclaration("}", NULL)) {
+    if (AtEnd()) {
+      AddError("Reached end of input in method options (missing '}').");
+      return false;
+    }
+
+    if (TryConsumeEndOfDeclaration(";", NULL)) {
+      // empty statement; ignore
+    } else {
+      LocationRecorder location(parent_location,
+                                optionsFieldNumber);
+      if (!ParseOption(mutable_options, location, OPTION_STATEMENT)) {
+        // This statement failed to parse.  Skip it, but keep looping to
+        // parse other statements.
+        SkipStatement();
       }
     }
-  } else {
-    DO(Consume(";"));
   }
 
   return true;
@@ -1406,35 +1532,47 @@
       if (!TryConsume(".")) break;
       file->mutable_package()->append(".");
     }
+
+    location.EndAt(input_->previous());
+
+    DO(ConsumeEndOfDeclaration(";", &location));
   }
 
-  DO(Consume(";"));
   return true;
 }
 
-bool Parser::ParseImport(string* import_filename,
-                         const LocationRecorder& root_location,
-                         int index) {
+bool Parser::ParseImport(RepeatedPtrField<string>* dependency,
+                         RepeatedField<int32>* public_dependency,
+                         RepeatedField<int32>* weak_dependency,
+                         const LocationRecorder& root_location) {
   DO(Consume("import"));
+  if (LookingAt("public")) {
+    LocationRecorder location(
+        root_location, FileDescriptorProto::kPublicDependencyFieldNumber,
+        public_dependency->size());
+    DO(Consume("public"));
+    *public_dependency->Add() = dependency->size();
+  } else if (LookingAt("weak")) {
+    LocationRecorder location(
+        root_location, FileDescriptorProto::kWeakDependencyFieldNumber,
+        weak_dependency->size());
+    DO(Consume("weak"));
+    *weak_dependency->Add() = dependency->size();
+  }
   {
     LocationRecorder location(root_location,
                               FileDescriptorProto::kDependencyFieldNumber,
-                              index);
-    DO(ConsumeString(import_filename,
+                              dependency->size());
+    DO(ConsumeString(dependency->Add(),
       "Expected a string naming the file to import."));
+
+    location.EndAt(input_->previous());
+
+    DO(ConsumeEndOfDeclaration(";", &location));
   }
-  DO(Consume(";"));
   return true;
 }
 
-bool Parser::ParseOption(Message* options,
-                         const LocationRecorder& options_location) {
-  DO(Consume("option"));
-  DO(ParseOptionAssignment(options, options_location));
-  DO(Consume(";"));
-  return true;
-}
-
 // ===================================================================
 
 SourceLocationTable::SourceLocationTable() {}