Subzero: Emit functions and global initializers in a separate thread.

src/IceGlobalContext.h

Index: src/IceGlobalContext.h
diff --git a/src/IceGlobalContext.h b/src/IceGlobalContext.h
index e3e0810cb02824af79d25403db76e50e9aedcc9b..a14a1df864d91515dc41d55098f2439a8c6adb58 100644
--- a/src/IceGlobalContext.h
+++ b/src/IceGlobalContext.h
@@ -23,6 +23,7 @@
 #include "IceClFlags.h"
 #include "IceIntrinsics.h"
 #include "IceRNG.h"
+#include "IceThreading.h"
 #include "IceTimerTree.h"
 #include "IceTypes.h"
 #include "IceUtils.h"
@@ -31,6 +32,7 @@ namespace Ice {
 
 class ClFlags;
 class ConstantPool;
+class EmitterWorkItem;
 class FuncSigType;
 
 // LockedPtr is a way to provide automatically locked access to some object.
@@ -276,18 +278,28 @@ public:
   void resetTimer(TimerStackIdT StackID);
   void setTimerName(TimerStackIdT StackID, const IceString &NewName);
 
+  // This is the first work item sequence number that the parser
+  // produces, and correspondingly the first sequence number that the
+  // emitter thread will wait for.  Start numbering at 1 to leave room
+  // for a sentinel, in case e.g. we wish to inject items with a
+  // special sequence number that may be executed out of order.
+  static uint32_t getFirstSequenceNumber() { return 1; }
   // Adds a newly parsed and constructed function to the Cfg work
   // queue.  Notifies any idle workers that a new function is
   // available for translating.  May block if the work queue is too
   // large, in order to control memory footprint.
-  void cfgQueueBlockingPush(std::unique_ptr<Cfg> Func);
+  void optQueueBlockingPush(std::unique_ptr<Cfg> Func);
   // Takes a Cfg from the work queue for translating.  May block if
   // the work queue is currently empty.  Returns nullptr if there is
   // no more work - the queue is empty and either end() has been
   // called or the Sequential flag was set.
-  std::unique_ptr<Cfg> cfgQueueBlockingPop();
+  std::unique_ptr<Cfg> optQueueBlockingPop();
   // Notifies that no more work will be added to the work queue.
-  void cfgQueueNotifyEnd() { CfgQ.notifyEnd(); }
+  void optQueueNotifyEnd() { OptQ.notifyEnd(); }
+
+  void emitQueueBlockingPush(EmitterWorkItem *Item);
+  EmitterWorkItem *emitQueueBlockingPop();
+  void emitQueueNotifyEnd() { EmitQ.notifyEnd(); }
 
   void startWorkerThreads() {
     size_t NumWorkers = getFlags().NumTranslationThreads;
@@ -300,18 +312,29 @@ public:
           &GlobalContext::translateFunctionsWrapper, this, WorkerTLS));
     }
     if (NumWorkers) {
-      // TODO(stichnot): start a new thread for the emitter queue worker.
+      ThreadContext *WorkerTLS = new ThreadContext();
+      Timers->initInto(WorkerTLS->Timers);
+      AllThreadContexts.push_back(WorkerTLS);
+      EmitterThreads.push_back(
+          std::thread(&GlobalContext::emitterWrapper, this, WorkerTLS));
     }
   }
 
   void waitForWorkerThreads() {
-    cfgQueueNotifyEnd();
-    // TODO(stichnot): call end() on the emitter work queue.
+    optQueueNotifyEnd();
     for (std::thread &Worker : TranslationThreads) {
       Worker.join();
     }
     TranslationThreads.clear();
-    // TODO(stichnot): join the emitter thread.
+
+    // Only notify the emit queue to end after all the translation
+    // threads have ended.
+    emitQueueNotifyEnd();
+    for (std::thread &Worker : EmitterThreads) {
+      Worker.join();
+    }
+    EmitterThreads.clear();
+
     if (ALLOW_DUMP) {
       auto Timers = getTimers();
       for (ThreadContext *TLS : AllThreadContexts)
@@ -334,6 +357,15 @@ public:
   // Translate functions from the Cfg queue until the queue is empty.
   void translateFunctions();
 
+  // Emitter thread startup routine.
+  void emitterWrapper(ThreadContext *MyTLS) {
+    ICE_TLS_SET_FIELD(TLS, MyTLS);
+    emitItems();
+  }
+  // Emit functions and global initializers from the emitter queue
+  // until the queue is empty.
+  void emitItems();
+
   // Utility function to match a symbol name against a match string.
   // This is used in a few cases where we want to take some action on
   // a particular function or symbol based on a command-line argument,
@@ -390,7 +422,8 @@ private:
   const ClFlags &Flags;
   RandomNumberGenerator RNG; // TODO(stichnot): Move into Cfg.
   std::unique_ptr<ELFObjectWriter> ObjectWriter;
-  BoundedProducerConsumerQueue<Cfg> CfgQ;
+  BoundedProducerConsumerQueue<Cfg> OptQ;
+  BoundedProducerConsumerQueue<EmitterWorkItem> EmitQ;
 
   LockedPtr<ArenaAllocator<>> getAllocator() {
     return LockedPtr<ArenaAllocator<>>(&Allocator, &AllocLock);
@@ -405,8 +438,9 @@ private:
     return LockedPtr<TimerList>(&Timers, &TimerLock);
   }
 
-  std::vector<ThreadContext *> AllThreadContexts;
-  std::vector<std::thread> TranslationThreads;
+  llvm::SmallVector<ThreadContext *, 128> AllThreadContexts;
+  llvm::SmallVector<std::thread, 128> TranslationThreads;
+  llvm::SmallVector<std::thread, 128> EmitterThreads;
   // Each thread has its own TLS pointer which is also held in
   // AllThreadContexts.
   ICE_TLS_DECLARE_FIELD(ThreadContext *, TLS);