Change LLVM bitcode linking to use tree reduction to scale better

llvm/tools/gold/gold-plugin.cpp

 //===-- gold-plugin.cpp - Plugin to gold for Link Time Optimization  ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This is a gold plugin for LLVM. It provides an LLVM implementation of the
@@ skipping 81 matching lines @@
   std::vector<std::string> DepLibs;
   // @LOCALMOD-END
 
   std::vector<sys::Path> Cleanup;
   lto_code_gen_t code_gen = NULL;
 }
 
 namespace options {
   enum generate_bc { BC_NO, BC_ALSO, BC_ONLY };
   static bool generate_api_file = false;
+  static bool gather_then_link = true; // @LOCALMOD
   static generate_bc generate_bc_file = BC_NO;
   static std::string bc_path;
   static std::string obj_path;
   static std::string extra_library_path;
   static std::string triple;
   static std::string mcpu;
   // Additional options to pass into the code generator.
   // Note: This array will contain all plugin options which are not claimed
   // as plugin exclusive to pass to the code generator.
   // For example, "generate-api-file" and "as"options are for the plugin
   // use only and will not be passed.
   static std::vector<std::string> extra;
 
   static void process_plugin_option(const char* opt_)
   {
     if (opt_ == NULL)
       return;
     llvm::StringRef opt = opt_;
 
     if (opt == "generate-api-file") {
       generate_api_file = true;
     } else if (opt.startswith("mcpu=")) {
       mcpu = opt.substr(strlen("mcpu="));
     } else if (opt.startswith("extra-library-path=")) {
       extra_library_path = opt.substr(strlen("extra_library_path="));
     } else if (opt.startswith("mtriple=")) {
       triple = opt.substr(strlen("mtriple="));
     } else if (opt.startswith("obj-path=")) {
       obj_path = opt.substr(strlen("obj-path="));
+      // @LOCALMOD-BEGIN
+    } else if (opt == "no-gather-then-link") {
+      gather_then_link = false;
+      // @LOCALMOD-END
     } else if (opt == "emit-llvm") {
       generate_bc_file = BC_ONLY;
     } else if (opt == "also-emit-llvm") {
       generate_bc_file = BC_ALSO;
     } else if (opt.startswith("also-emit-llvm=")) {
       llvm::StringRef path = opt.substr(strlen("also-emit-llvm="));
       generate_bc_file = BC_ALSO;
       if (!bc_path.empty()) {
         (*message)(LDPL_WARNING, "Path to the output IL file specified twice. "
                    "Discarding %s", opt_);
@@ skipping 313 matching lines @@
   // @LOCALMOD-BEGIN
   // Do not merge the module if it's a PSO.
   // If the PSO's soname is set, add it to DepLibs.
   cf.is_linked_in = false;
   if (code_gen) {
     if (is_shared) {
       if (soname && strlen(soname) > 0) {
         DepLibs.push_back(soname);
       }
     } else {
-      lto_codegen_add_module(code_gen, M);
+      if (options::gather_then_link) {
+        lto_codegen_gather_module_for_link(code_gen, M);
+      } else {
+        lto_codegen_add_module(code_gen, M);
+      }
       cf.is_linked_in = true;
     }
   }
+
+  // With gather_then_link, the modules are disposed when linking.
+  if (!options::gather_then_link)
+    lto_module_dispose(M);
   // @LOCALMOD-END
 
-  lto_module_dispose(M);
-
   return LDPS_OK;
 }
 
 /// all_symbols_read_hook - gold informs us that all symbols have been read.
 /// At this point, we use get_symbols to see if any of our definitions have
 /// been overridden by a native object file. Then, perform optimization and
 /// codegen.
 static ld_plugin_status all_symbols_read_hook(void) {
   std::ofstream api_file;
   assert(code_gen);
 
+  // @LOCALMOD-BEGIN
+  if (options::gather_then_link) {
+    lto_codegen_link_gathered_modules_and_dispose(code_gen);
+  }
+  // @LOCALMOD-END
+
   if (options::generate_api_file) {
     api_file.open("apifile.txt", std::ofstream::out | std::ofstream::trunc);
     if (!api_file.is_open()) {
       (*message)(LDPL_FATAL, "Unable to open apifile.txt for writing.");
       abort();
     }
   }
 
   // If we don't preserve any symbols, libLTO will assume that all symbols are
   // needed. Keep all symbols unless we're producing a final executable.
@@ skipping 174 matching lines @@
 static ld_plugin_status cleanup_hook(void) {
   std::string ErrMsg;
 
   for (int i = 0, e = Cleanup.size(); i != e; ++i)
     if (Cleanup[i].eraseFromDisk(false, &ErrMsg))
       (*message)(LDPL_ERROR, "Failed to delete '%s': %s", Cleanup[i].c_str(),
                  ErrMsg.c_str());
 
   return LDPS_OK;
 }