Allow Subzero to parse function blocks in parallel.

src/IceGlobalContext.cpp

 //===- subzero/src/IceGlobalContext.cpp - Global context defs -------------===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
@@ skipping 200 matching lines @@
   TypePool<IceType_i8, int8_t, ConstantInteger32> Integers8;
   TypePool<IceType_i16, int16_t, ConstantInteger32> Integers16;
   TypePool<IceType_i32, int32_t, ConstantInteger32> Integers32;
   TypePool<IceType_i64, int64_t, ConstantInteger64> Integers64;
   TypePool<IceType_i32, RelocatableTuple, ConstantRelocatable> Relocatables;
   TypePool<IceType_i32, RelocatableTuple, ConstantRelocatable>
       ExternRelocatables;
   UndefPool Undefs;
 };
 
+void GlobalContext::waitForWorkerThreads() {
+  if (WaitForWorkerThreadsCalled.exchange(true))
+    return;
+  optQueueNotifyEnd();
+  for (std::thread &Worker : TranslationThreads) {
+    Worker.join();
+  }
+  TranslationThreads.clear();
+
+  // Only notify the emit queue to end after all the translation threads have
+  // ended.
+  emitQueueNotifyEnd();
+  for (std::thread &Worker : EmitterThreads) {
+    Worker.join();
+  }
+  EmitterThreads.clear();
+
+  if (BuildDefs::timers()) {
+    auto Timers = getTimers();
+    for (ThreadContext *TLS : AllThreadContexts)
+      Timers->mergeFrom(TLS->Timers);
+  }
+  if (BuildDefs::dump()) {
+    // Do a separate loop over AllThreadContexts to avoid holding two locks

[Jim Stichnoth, 2016/03/31 16:08:08]

reflow...

[Karl, 2016/03/31 16:33:49]

Done.

+    // at once.
+    auto Stats = getStatsCumulative();
+    for (ThreadContext *TLS : AllThreadContexts)
+      Stats->add(TLS->StatsCumulative);
+  }
+}
+
 void GlobalContext::CodeStats::dump(const IceString &Name, GlobalContext *Ctx) {
   if (!BuildDefs::dump())
     return;
   OstreamLocker _(Ctx);
   Ostream &Str = Ctx->getStrDump();
 #define X(str, tag)                                                            \
   Str << "|" << Name << "|" str "|" << Stats[CS_##tag] << "\n";
   CODESTATS_TABLE
 #undef X
   Str << "|" << Name << "|Spills+Fills|"
@@ skipping 16 matching lines @@
     Str << "|i64=" << Pool->Integers64.size();
     Str << "|Rel=" << Pool->Relocatables.size();
     Str << "|ExtRel=" << Pool->ExternRelocatables.size();
   }
   Str << "\n";
 }
 
 GlobalContext::GlobalContext(Ostream *OsDump, Ostream *OsEmit, Ostream *OsError,
                              ELFStreamer *ELFStr)
     : ConstPool(new ConstantPool()), ErrorStatus(), StrDump(OsDump),
-      StrEmit(OsEmit), StrError(OsError), ObjectWriter(),
-      OptQ(/*Sequential=*/Flags.isSequential(),
-           /*MaxSize=*/Flags.getNumTranslationThreads()),
+      StrEmit(OsEmit), StrError(OsError), WaitForWorkerThreadsCalled(false),
+      ObjectWriter(), OptQ(/*Sequential=*/Flags.isSequential(),
+                           /*MaxSize=*/
+                           (Flags.getParseParallel() && Flags.getBuildOnRead())
+                               ? MaxOptQSize
+                               : Flags.getNumTranslationThreads()),
       // EmitQ is allowed unlimited size.
       EmitQ(/*Sequential=*/Flags.isSequential()),
       DataLowering(TargetDataLowering::createLowering(this)) {
   assert(OsDump && "OsDump is not defined for GlobalContext");
   assert(OsEmit && "OsEmit is not defined for GlobalContext");
   assert(OsError && "OsError is not defined for GlobalContext");
   // Make sure thread_local fields are properly initialized before any
   // accesses are made.  Do this here instead of at the start of
   // main() so that all clients (e.g. unit tests) can benefit for
   // free.
@@ skipping 32 matching lines @@
   RuntimeHelperFunc[static_cast<size_t>(RuntimeHelper::H_##Tag)] =             \
       getConstantExternSym(Name);
   RUNTIME_HELPER_FUNCTIONS_TABLE
 #undef X
 
   TargetLowering::staticInit(this);
 }
 
 void GlobalContext::translateFunctions() {
   TimerMarker Timer(TimerStack::TT_translateFunctions, this);
-  while (std::unique_ptr<Cfg> Func = optQueueBlockingPop()) {
+  while (std::unique_ptr<OptWorkItem> OptItem = optQueueBlockingPop()) {
+    std::unique_ptr<Cfg> Func = OptItem->getParsedCfg();

[John, 2016/03/31 15:51:36]

optional: auto

[Karl, 2016/03/31 16:33:49]

Done.

     // Install Func in TLS for Cfg-specific container allocators.
     CfgLocalAllocatorScope _(Func.get());
     // Reset per-function stats being accumulated in TLS.
     resetStats();
     // Set verbose level to none if the current function does NOT
     // match the -verbose-focus command-line option.
     if (!matchSymbolName(Func->getFunctionName(),
                          getFlags().getVerboseFocusOn()))
       Func->setVerbose(IceV_None);
     // Disable translation if -notranslate is specified, or if the
@@ skipping 537 matching lines @@
                                  const IceString &NewName) {
   auto Timers = &ICE_TLS_GET_FIELD(TLS)->Timers;
   assert(StackID < Timers->size());
   Timers->at(StackID).setName(NewName);
 }
 
 // Note: optQueueBlockingPush and optQueueBlockingPop use unique_ptr at the
 // interface to take and transfer ownership, but they internally store the raw
 // Cfg pointer in the work queue. This allows e.g. future queue optimizations
 // such as the use of atomics to modify queue elements.
-void GlobalContext::optQueueBlockingPush(std::unique_ptr<Cfg> Func) {
-  assert(Func);
+void GlobalContext::optQueueBlockingPush(std::unique_ptr<OptWorkItem> Item) {
+  assert(Item);
   {
     TimerMarker _(TimerStack::TT_qTransPush, this);
-    OptQ.blockingPush(std::move(Func));
+    OptQ.blockingPush(std::move(Item));
   }
   if (getFlags().isSequential())
     translateFunctions();
 }
 
-std::unique_ptr<Cfg> GlobalContext::optQueueBlockingPop() {
+std::unique_ptr<OptWorkItem> GlobalContext::optQueueBlockingPop() {
   TimerMarker _(TimerStack::TT_qTransPop, this);
-  return std::unique_ptr<Cfg>(OptQ.blockingPop());
+  return std::unique_ptr<OptWorkItem>(OptQ.blockingPop());
 }
 
 void GlobalContext::emitQueueBlockingPush(
     std::unique_ptr<EmitterWorkItem> Item) {
   assert(Item);
   {
     TimerMarker _(TimerStack::TT_qEmitPush, this);
     EmitQ.blockingPush(std::move(Item));
   }
   if (getFlags().isSequential())
@@ skipping 54 matching lines @@
   Ctx = Func->getContext();
   Active =
       Func->getFocusedTiming() || Ctx->getFlags().getSubzeroTimingEnabled();
   if (Active)
     Ctx->pushTimer(ID, StackID);
 }
 
 ICE_TLS_DEFINE_FIELD(GlobalContext::ThreadContext *, GlobalContext, TLS);
 
 } // end of namespace Ice