Subzero: Emit functions and global initializers in a separate thread.

src/IceGlobalContext.h

 //===- subzero/src/IceGlobalContext.h - Global context defs -----*- C++ -*-===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file declares aspects of the compilation that persist across
 // multiple functions.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef SUBZERO_SRC_ICEGLOBALCONTEXT_H
 #define SUBZERO_SRC_ICEGLOBALCONTEXT_H
 
 #include <array>
 #include <mutex>
 #include <thread>
 
 #include "IceDefs.h"
 #include "IceClFlags.h"
 #include "IceIntrinsics.h"
 #include "IceRNG.h"
+#include "IceThreading.h"
 #include "IceTimerTree.h"
 #include "IceTypes.h"
 #include "IceUtils.h"
 
 namespace Ice {
 
 class ClFlags;
 class ConstantPool;
+class EmitterWorkItem;
 class FuncSigType;
 
 // LockedPtr is a way to provide automatically locked access to some object.
 template <typename T> class LockedPtr {
   LockedPtr() = delete;
   LockedPtr(const LockedPtr &) = delete;
   LockedPtr &operator=(const LockedPtr &) = delete;
 
 public:
   LockedPtr(T *Value, GlobalLockType *Lock) : Value(Value), Lock(Lock) {
@@ skipping 225 matching lines @@
   void dumpTimers(TimerStackIdT StackID = TSK_Default,
                   bool DumpCumulative = true);
   // The following methods affect only the calling thread's TLS timer
   // data.
   TimerIdT getTimerID(TimerStackIdT StackID, const IceString &Name);
   void pushTimer(TimerIdT ID, TimerStackIdT StackID);
   void popTimer(TimerIdT ID, TimerStackIdT StackID);
   void resetTimer(TimerStackIdT StackID);
   void setTimerName(TimerStackIdT StackID, const IceString &NewName);
 
+  // This is the first work item sequence number that the parser
+  // produces, and correspondingly the first sequence number that the
+  // emitter thread will wait for.  Start numbering at 1 to leave room
+  // for a sentinel, in case e.g. we wish to inject items with a
+  // special sequence number that may be executed out of order.
+  static uint32_t getFirstSequenceNumber() { return 1; }
   // Adds a newly parsed and constructed function to the Cfg work
   // queue.  Notifies any idle workers that a new function is
   // available for translating.  May block if the work queue is too
   // large, in order to control memory footprint.
-  void cfgQueueBlockingPush(std::unique_ptr<Cfg> Func);
+  void optQueueBlockingPush(std::unique_ptr<Cfg> Func);
   // Takes a Cfg from the work queue for translating.  May block if
   // the work queue is currently empty.  Returns nullptr if there is
   // no more work - the queue is empty and either end() has been
   // called or the Sequential flag was set.
-  std::unique_ptr<Cfg> cfgQueueBlockingPop();
+  std::unique_ptr<Cfg> optQueueBlockingPop();
   // Notifies that no more work will be added to the work queue.
-  void cfgQueueNotifyEnd() { CfgQ.notifyEnd(); }
+  void optQueueNotifyEnd() { OptQ.notifyEnd(); }
+
+  void emitQueueBlockingPush(EmitterWorkItem *Item);
+  EmitterWorkItem *emitQueueBlockingPop();
+  void emitQueueNotifyEnd() { EmitQ.notifyEnd(); }
 
   void startWorkerThreads() {
     size_t NumWorkers = getFlags().getNumTranslationThreads();
     auto Timers = getTimers();
     for (size_t i = 0; i < NumWorkers; ++i) {
       ThreadContext *WorkerTLS = new ThreadContext();
       Timers->initInto(WorkerTLS->Timers);
       AllThreadContexts.push_back(WorkerTLS);
       TranslationThreads.push_back(std::thread(
           &GlobalContext::translateFunctionsWrapper, this, WorkerTLS));
     }
     if (NumWorkers) {
-      // TODO(stichnot): start a new thread for the emitter queue worker.
+      ThreadContext *WorkerTLS = new ThreadContext();
+      Timers->initInto(WorkerTLS->Timers);
+      AllThreadContexts.push_back(WorkerTLS);
+      EmitterThreads.push_back(
+          std::thread(&GlobalContext::emitterWrapper, this, WorkerTLS));
     }
   }
 
   void waitForWorkerThreads() {
-    cfgQueueNotifyEnd();
-    // TODO(stichnot): call end() on the emitter work queue.
+    optQueueNotifyEnd();
     for (std::thread &Worker : TranslationThreads) {
       Worker.join();
     }
     TranslationThreads.clear();
-    // TODO(stichnot): join the emitter thread.
+
+    // Only notify the emit queue to end after all the translation
+    // threads have ended.
+    emitQueueNotifyEnd();
+    for (std::thread &Worker : EmitterThreads) {
+      Worker.join();
+    }
+    EmitterThreads.clear();
+
     if (ALLOW_DUMP) {
       auto Timers = getTimers();
       for (ThreadContext *TLS : AllThreadContexts)
         Timers->mergeFrom(TLS->Timers);
     }
     if (ALLOW_DUMP) {
       // Do a separate loop over AllThreadContexts to avoid holding
       // two locks at once.
       auto Stats = getStatsCumulative();
       for (ThreadContext *TLS : AllThreadContexts)
         Stats->add(TLS->StatsCumulative);
     }
   }
 
   // Translation thread startup routine.
   void translateFunctionsWrapper(ThreadContext *MyTLS) {
     ICE_TLS_SET_FIELD(TLS, MyTLS);
     translateFunctions();
   }
   // Translate functions from the Cfg queue until the queue is empty.
   void translateFunctions();
 
+  // Emitter thread startup routine.
+  void emitterWrapper(ThreadContext *MyTLS) {
+    ICE_TLS_SET_FIELD(TLS, MyTLS);
+    emitItems();
+  }
+  // Emit functions and global initializers from the emitter queue
+  // until the queue is empty.
+  void emitItems();
+
   // Utility function to match a symbol name against a match string.
   // This is used in a few cases where we want to take some action on
   // a particular function or symbol based on a command-line argument,
   // such as changing the verbose level for a particular function.  An
   // empty Match argument means match everything.  Returns true if
   // there is a match.
   static bool matchSymbolName(const IceString &SymbolName,
                               const IceString &Match) {
     return Match.empty() || Match == SymbolName;
   }
@@ skipping 36 matching lines @@
   ICE_CACHELINE_BOUNDARY;
 
   const VerboseMask VMask;
   Intrinsics IntrinsicsInfo;
   const TargetArch Arch;
   const OptLevel Opt;
   const IceString TestPrefix;
   const ClFlags &Flags;
   RandomNumberGenerator RNG; // TODO(stichnot): Move into Cfg.
   std::unique_ptr<ELFObjectWriter> ObjectWriter;
-  BoundedProducerConsumerQueue<Cfg> CfgQ;
+  BoundedProducerConsumerQueue<Cfg> OptQ;
+  BoundedProducerConsumerQueue<EmitterWorkItem> EmitQ;
 
   LockedPtr<ArenaAllocator<>> getAllocator() {
     return LockedPtr<ArenaAllocator<>>(&Allocator, &AllocLock);
   }
   LockedPtr<ConstantPool> getConstPool() {
     return LockedPtr<ConstantPool>(ConstPool.get(), &ConstPoolLock);
   }
   LockedPtr<CodeStats> getStatsCumulative() {
     return LockedPtr<CodeStats>(&StatsCumulative, &StatsLock);
   }
   LockedPtr<TimerList> getTimers() {
     return LockedPtr<TimerList>(&Timers, &TimerLock);
   }
 
-  std::vector<ThreadContext *> AllThreadContexts;
-  std::vector<std::thread> TranslationThreads;
+  llvm::SmallVector<ThreadContext *, 128> AllThreadContexts;
+  llvm::SmallVector<std::thread, 128> TranslationThreads;
+  llvm::SmallVector<std::thread, 128> EmitterThreads;
   // Each thread has its own TLS pointer which is also held in
   // AllThreadContexts.
   ICE_TLS_DECLARE_FIELD(ThreadContext *, TLS);
 
   // Private helpers for mangleName()
   typedef llvm::SmallVector<char, 32> ManglerVector;
   void incrementSubstitutions(ManglerVector &OldName) const;
 
 public:
   static void TlsInit() { ICE_TLS_INIT_FIELD(TLS); }
@@ skipping 47 matching lines @@
   explicit OstreamLocker(GlobalContext *Ctx) : Ctx(Ctx) { Ctx->lockStr(); }
   ~OstreamLocker() { Ctx->unlockStr(); }
 
 private:
   GlobalContext *const Ctx;
 };
 
 } // end of namespace Ice
 
 #endif // SUBZERO_SRC_ICEGLOBALCONTEXT_H