Subzero: Initial implementation of multithreaded translation.

src/IceGlobalContext.h

Index: src/IceGlobalContext.h
diff --git a/src/IceGlobalContext.h b/src/IceGlobalContext.h
index 1f0a600d485fa8dcba7d3c5915d0bdc936e0428d..6bb1c0d2ce5ee624daf4687b7ea50d23fa37eb70 100644
--- a/src/IceGlobalContext.h
+++ b/src/IceGlobalContext.h
@@ -15,8 +15,9 @@
 #ifndef SUBZERO_SRC_ICEGLOBALCONTEXT_H
 #define SUBZERO_SRC_ICEGLOBALCONTEXT_H
 
-#include <memory>
 #include <mutex>
+#include <queue>
+#include <thread>
 
 #include "IceDefs.h"
 #include "IceClFlags.h"
@@ -24,6 +25,7 @@
 #include "IceRNG.h"
 #include "IceTimerTree.h"
 #include "IceTypes.h"
+#include "IceUtils.h"
 
 namespace Ice {
 
@@ -31,8 +33,6 @@ class ClFlags;
 class ConstantPool;
 class FuncSigType;
 
-typedef std::mutex GlobalLockType;
-
 // LockedPtr is a way to provide automatically locked access to some object.
 template <typename T> class LockedPtr {
   LockedPtr() = delete;
@@ -102,14 +102,7 @@ public:
                 IceString TestPrefix, const ClFlags &Flags);
   ~GlobalContext();
 
-  // Returns true if any of the specified options in the verbose mask
-  // are set.  If the argument is omitted, it checks if any verbose
-  // options at all are set.
   VerboseMask getVerbose() const { return VMask; }
-  bool isVerbose(VerboseMask Mask = IceV_All) const { return VMask & Mask; }
-  void setVerbose(VerboseMask Mask) { VMask = Mask; }
-  void addVerbose(VerboseMask Mask) { VMask |= Mask; }
-  void subVerbose(VerboseMask Mask) { VMask &= ~Mask; }
 
   // The dump and emit streams need to be used by only one thread at a
   // time.  This is done by exclusively reserving the streams via
@@ -129,6 +122,9 @@ public:
 
   TargetArch getTargetArch() const { return Arch; }
   OptLevel getOptLevel() const { return Opt; }
+  LockedPtr<std::error_code> getErrorStatus() {
+    return LockedPtr<std::error_code>(&ErrorStatus, &ErrorStatusLock);
+  }
 
   // When emitting assembly, we allow a string to be prepended to
   // names of translated functions.  This makes it easier to create an
@@ -229,34 +225,99 @@ public:
   void dumpTimers(TimerStackIdT StackID = TSK_Default,
                   bool DumpCumulative = true);
 
+  // Adds a newly parsed and constructed function to the Cfg work
+  // queue.  Notifies any idle workers that a new function is
+  // available for translating.  May block if the work queue is too
+  // large, in order to control memory footprint.
+  void cfgQueueAdd(Cfg *Func) { CfgQ.blockingPush(Func); }
+  // Takes a Cfg from the work queue for translating.  May block if
+  // the work queue is currently empty.  Returns nullptr if there is
+  // no more work - the queue is empty and either end() has been
+  // called or the Sequential flag was set.
+  Cfg *cfgQueueGet() { return CfgQ.blockingPop(); }

[JF, 2015/01/26 17:54:50]

I'd change these names too (add/get to blockingPush/bloxkingPop).

[Jim Stichnoth, 2015/01/27 00:56:18]

Done.

+  // Notifies that no more work will be added to the work queue.
+  void cfgQueueEnd() { CfgQ.end(); }

[JF, 2015/01/26 17:54:51]

Now that I think of it, notifyEnded would probably be clearer.

[Jim Stichnoth, 2015/01/27 00:56:18]

Done.

+
+  void startWorkerThreads() {
+    size_t NumWorkers = getFlags().NumTranslationThreads;
+    for (size_t i = 0; i < NumWorkers; ++i) {
+      ThreadContext *WorkerTLS = new ThreadContext();
+      AllThreadContexts.push_back(WorkerTLS);
+      TranslationThreads.push_back(std::thread(
+          &GlobalContext::translateFunctionsWrapper, this, WorkerTLS));
+    }
+    if (NumWorkers) {
+      // TODO(stichnot): start a new thread for the emitter queue worker.
+    }
+  }
+
+  void waitForWorkerThreads() {
+    cfgQueueEnd();
+    // TODO(stichnot): call end() on the emitter work queue.
+    for (std::thread &Worker : TranslationThreads) {
+      Worker.join();
+    }
+    TranslationThreads.clear();
+    // TODO(stichnot): join the emitter thread.
+  }
+
+  // Translation thread startup routine.
+  void translateFunctionsWrapper(ThreadContext *MyTLS) {
+    TLS = MyTLS;
+    translateFunctions();
+  }
+  // Translate functions from the Cfg queue until the queue is empty.
+  void translateFunctions();
+
+  // Utility function to match a symbol name against a match string.
+  // This is used in a few cases where we want to take some action on
+  // a particular function or symbol based on a command-line argument,
+  // such as changing the verbose level for a particular function.  An
+  // empty Match argument means match everything.  Returns true if
+  // there is a match.
+  static bool matchSymbolName(const IceString &SymbolName,
+                              const IceString &Match) {
+    return Match.empty() || Match == SymbolName;
+  }
+
 private:
-  // Try to make sure the mutexes are allocated on separate cache
-  // lines, assuming the maximum cache line size is 64.
-  const static size_t MaxCacheLineSize = 64;
+  // Try to ensure mutexes are allocated on separate cache lines.
+
+  // Managed by getAllocator()
   alignas(MaxCacheLineSize) GlobalLockType AllocLock;
+  ArenaAllocator<> Allocator;
+
+  // Managed by getConstantPool()
   alignas(MaxCacheLineSize) GlobalLockType ConstPoolLock;
+  std::unique_ptr<ConstantPool> ConstPool;
+
+  // Managed by getErrorStatus()
+  alignas(MaxCacheLineSize) GlobalLockType ErrorStatusLock;
+  std::error_code ErrorStatus;
+
+  // Managed by getStatsCumulative()
   alignas(MaxCacheLineSize) GlobalLockType StatsLock;
+  CodeStats StatsCumulative;
+
+  // Managed by getTimers()
   alignas(MaxCacheLineSize) GlobalLockType TimerLock;
+  std::vector<TimerStack> Timers;
 
   // StrLock is a global lock on the dump and emit output streams.
   typedef std::mutex StrLockType;
-  StrLockType StrLock;
-
+  alignas(MaxCacheLineSize) StrLockType StrLock;
   Ostream *StrDump; // Stream for dumping / diagnostics
   Ostream *StrEmit; // Stream for code emission
 
-  ArenaAllocator<> Allocator;
-  VerboseMask VMask;
-  std::unique_ptr<ConstantPool> ConstPool;
+  alignas(MaxCacheLineSize) const VerboseMask VMask;
   Intrinsics IntrinsicsInfo;
   const TargetArch Arch;
   const OptLevel Opt;
   const IceString TestPrefix;
   const ClFlags &Flags;
-  RandomNumberGenerator RNG;
+  RandomNumberGenerator RNG; // TODO(stichnot): Move into Cfg.
   std::unique_ptr<ELFObjectWriter> ObjectWriter;
-  CodeStats StatsCumulative;
-  std::vector<TimerStack> Timers;
+  BoundedProducerConsumerQueue<Cfg> CfgQ;
 
   LockedPtr<ArenaAllocator<>> getAllocator() {
     return LockedPtr<ArenaAllocator<>>(&Allocator, &AllocLock);
@@ -272,6 +333,7 @@ private:
   }
 
   std::vector<ThreadContext *> AllThreadContexts;
+  std::vector<std::thread> TranslationThreads;
   // Each thread has its own TLS pointer which is also held in
   // AllThreadContexts.
   ICE_ATTRIBUTE_TLS static ThreadContext *TLS;