third_party/protobuf: Update to HEAD (83d681ee2c)

third_party/protobuf/src/google/protobuf/compiler/python/python_generator.cc

Index: third_party/protobuf/src/google/protobuf/compiler/python/python_generator.cc
diff --git a/third_party/protobuf/src/google/protobuf/compiler/python/python_generator.cc b/third_party/protobuf/src/google/protobuf/compiler/python/python_generator.cc
index 0553dd0d9d37456e0255cc80ecafbfce503f2583..3b1b0d1db734fa4c925b62576de6525b32c771ac 100644
--- a/third_party/protobuf/src/google/protobuf/compiler/python/python_generator.cc
+++ b/third_party/protobuf/src/google/protobuf/compiler/python/python_generator.cc
@@ -44,6 +44,7 @@
 // performance-minded Python code leverage the fast C++ implementation
 // directly.
 
+#include <algorithm>
 #include <google/protobuf/stubs/hash.h>
 #include <limits>
 #include <map>
@@ -87,8 +88,8 @@ string StripProto(const string& filename) {
 // Returns the Python module name expected for a given .proto filename.
 string ModuleName(const string& filename) {
   string basename = StripProto(filename);
-  StripString(&basename, "-", '_');
-  StripString(&basename, "/", '.');
+  ReplaceCharacters(&basename, "-", '_');
+  ReplaceCharacters(&basename, "/", '.');
   return basename + "_pb2";
 }
 
@@ -107,20 +108,25 @@ string ModuleAlias(const string& filename) {
   return module_name;
 }
 
-
-// Returns an import statement of form "from X.Y.Z import T" for the given
-// .proto filename.
-string ModuleImportStatement(const string& filename) {
-  string module_name = ModuleName(filename);
-  int last_dot_pos = module_name.rfind('.');
-  if (last_dot_pos == string::npos) {
-    // NOTE(petya): this is not tested as it would require a protocol buffer
-    // outside of any package, and I don't think that is easily achievable.
-    return "import " + module_name;
-  } else {
-    return "from " + module_name.substr(0, last_dot_pos) + " import " +
-        module_name.substr(last_dot_pos + 1);
+// Keywords reserved by the Python language.
+const char* const kKeywords[] = {
+    "False",   "None",     "True",     "and",    "as",    "assert", "break",
+    "class",   "continue", "def",      "del",    "elif",  "else",   "except",
+    "finally", "for",      "from",     "global", "if",    "import", "in",
+    "is",      "lambda",   "nonlocal", "not",    "or",    "pass",   "raise",
+    "return",  "try",      "while",    "with",   "yield",
+};
+const char* const* kKeywordsEnd =
+    kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0]));
+
+bool ContainsPythonKeyword(const string& module_name) {
+  vector<string> tokens = Split(module_name, ".");
+  for (int i = 0; i < tokens.size(); ++i) {
+    if (std::find(kKeywords, kKeywordsEnd, tokens[i]) != kKeywordsEnd) {
+      return true;
+    }
   }
+  return false;
 }
 
 
@@ -314,7 +320,7 @@ bool Generator::Generate(const FileDescriptor* file,
   file_ = file;
   string module_name = ModuleName(file->name());
   string filename = module_name;
-  StripString(&filename, ".", '/');
+  ReplaceCharacters(&filename, ".", '/');
   filename += ".py";
 
   FileDescriptorProto fdp;
@@ -359,10 +365,32 @@ bool Generator::Generate(const FileDescriptor* file,
 void Generator::PrintImports() const {
   for (int i = 0; i < file_->dependency_count(); ++i) {
     const string& filename = file_->dependency(i)->name();
-    string import_statement = ModuleImportStatement(filename);
+
+    string module_name = ModuleName(filename);
     string module_alias = ModuleAlias(filename);
-    printer_->Print("$statement$ as $alias$\n", "statement",
-                    import_statement, "alias", module_alias);
+    if (ContainsPythonKeyword(module_name)) {
+      // If the module path contains a Python keyword, we have to quote the
+      // module name and import it using importlib. Otherwise the usual kind of
+      // import statement would result in a syntax error from the presence of
+      // the keyword.
+      printer_->Print("import importlib\n");
+      printer_->Print("$alias$ = importlib.import_module('$name$')\n", "alias",
+                      module_alias, "name", module_name);
+    } else {
+      int last_dot_pos = module_name.rfind('.');
+      string import_statement;
+      if (last_dot_pos == string::npos) {
+        // NOTE(petya): this is not tested as it would require a protocol buffer
+        // outside of any package, and I don't think that is easily achievable.
+        import_statement = "import " + module_name;
+      } else {
+        import_statement = "from " + module_name.substr(0, last_dot_pos) +
+                           " import " + module_name.substr(last_dot_pos + 1);
+      }
+      printer_->Print("$statement$ as $alias$\n", "statement", import_statement,
+                      "alias", module_alias);
+    }
+
     CopyPublicDependenciesAliases(module_alias, file_->dependency(i));
   }
   printer_->Print("\n");
@@ -402,6 +430,15 @@ void Generator::PrintFileDescriptor() const {
     }
     printer_->Print("]");
   }
+  if (file_->public_dependency_count() > 0) {
+    printer_->Print(",\npublic_dependencies=[");
+    for (int i = 0; i < file_->public_dependency_count(); ++i) {
+      string module_alias = ModuleAlias(file_->public_dependency(i)->name());
+      printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
+                      module_alias);
+    }
+    printer_->Print("]");
+  }
 
   // TODO(falk): Also print options and fix the message_type, enum_type,
   //             service and extension later in the generation.
@@ -707,11 +744,18 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const {
     m["name"] = desc->name();
     m["full_name"] = desc->full_name();
     m["index"] = SimpleItoa(desc->index());
+    string options_string =
+        OptionsValue("OneofOptions", desc->options().SerializeAsString());
+    if (options_string == "None") {
+      m["options"] = "";
+    } else {
+      m["options"] = ", options=" + options_string;
+    }
     printer_->Print(
         m,
         "_descriptor.OneofDescriptor(\n"
         "  name='$name$', full_name='$full_name$',\n"
-        "  index=$index$, containing_type=None, fields=[]),\n");
+        "  index=$index$, containing_type=None, fields=[]$options$),\n");
   }
   printer_->Outdent();
   printer_->Print("],\n");
@@ -1059,6 +1103,8 @@ void Generator::PrintFieldDescriptor(
   m["default_value"] = StringifyDefaultValue(field);
   m["is_extension"] = is_extension ? "True" : "False";
   m["options"] = OptionsValue("FieldOptions", options_string);
+  m["json_name"] = field.has_json_name() ?
+      ", json_name='" + field.json_name() + "'": "";
   // We always set message_type and enum_type to None at this point, and then
   // these fields in correctly after all referenced descriptors have been
   // defined and/or imported (see FixForeignFieldsInDescriptors()).
@@ -1069,7 +1115,7 @@ void Generator::PrintFieldDescriptor(
     "  has_default_value=$has_default_value$, default_value=$default_value$,\n"
     "  message_type=None, enum_type=None, containing_type=None,\n"
     "  is_extension=$is_extension$, extension_scope=None,\n"
-    "  options=$options$)";
+    "  options=$options$$json_name$)";
   printer_->Print(m, field_descriptor_decl);
 }
 
@@ -1235,6 +1281,18 @@ void Generator::FixAllDescriptorOptions() const {
   }
 }
 
+void Generator::FixOptionsForOneof(const OneofDescriptor& oneof) const {
+  string oneof_options = OptionsValue(
+      "OneofOptions", oneof.options().SerializeAsString());
+  if (oneof_options != "None") {
+    string oneof_name = strings::Substitute(
+        "$0.$1['$2']",
+        ModuleLevelDescriptorName(*oneof.containing_type()),
+        "oneofs_by_name", oneof.name());
+    PrintDescriptorOptionsFixingCode(oneof_name, oneof_options, printer_);
+  }
+}
+
 // Prints expressions that set the options for an enum descriptor and its
 // value descriptors.
 void Generator::FixOptionsForEnum(const EnumDescriptor& enum_descriptor) const {
@@ -1288,6 +1346,10 @@ void Generator::FixOptionsForMessage(const Descriptor& descriptor) const {
   for (int i = 0; i < descriptor.nested_type_count(); ++i) {
     FixOptionsForMessage(*descriptor.nested_type(i));
   }
+  // Oneofs.
+  for (int i = 0; i < descriptor.oneof_decl_count(); ++i) {
+    FixOptionsForOneof(*descriptor.oneof_decl(i));
+  }
   // Enums.
   for (int i = 0; i < descriptor.enum_type_count(); ++i) {
     FixOptionsForEnum(*descriptor.enum_type(i));