Allow Subzero to parse function blocks in parallel.

src/IceGlobalContext.h

Index: src/IceGlobalContext.h
diff --git a/src/IceGlobalContext.h b/src/IceGlobalContext.h
index 6c6c22248c6fafd2cf5490c0c82cc545f9060484..3d9a9c38a23b136bd1f6867fa1a972bbb59eda78 100644
--- a/src/IceGlobalContext.h
+++ b/src/IceGlobalContext.h
@@ -77,6 +77,21 @@ private:
   GlobalLockType *Lock;
 };
 
+/// OptWorkItem is a simple wrapper used to pass parse information on a function
+/// block, to a translator thread.
+class OptWorkItem {
+  OptWorkItem(const OptWorkItem &) = delete;
+  OptWorkItem &operator=(const OptWorkItem &) = delete;
+
+public:
+  // Get the Cfg for the funtion to translate.
+  virtual std::unique_ptr<Cfg> getParsedCfg() = 0;
+  virtual ~OptWorkItem() {}
+
+protected:
+  OptWorkItem() {}

[Jim Stichnoth, 2016/03/25 04:31:04]

OptWorkItem() = default;
?

Same for ~OptWorkItem() ?

[Karl, 2016/03/29 17:35:02]

Done.

+};
+
 class GlobalContext {
   GlobalContext() = delete;
   GlobalContext(const GlobalContext &) = delete;
@@ -378,12 +393,12 @@ public:
   /// Notifies any idle workers that a new function is available for
   /// translating. May block if the work queue is too large, in order to control
   /// memory footprint.
-  void optQueueBlockingPush(std::unique_ptr<Cfg> Func);
+  void optQueueBlockingPush(std::unique_ptr<OptWorkItem> Item);
   /// Takes a Cfg from the work queue for translating. May block if the work
   /// queue is currently empty. Returns nullptr if there is no more work - the
   /// queue is empty and either end() has been called or the Sequential flag was
   /// set.
-  std::unique_ptr<Cfg> optQueueBlockingPop();
+  std::unique_ptr<OptWorkItem> optQueueBlockingPop();
   /// Notifies that no more work will be added to the work queue.
   void optQueueNotifyEnd() { OptQ.notifyEnd(); }
 
@@ -425,34 +440,7 @@ public:
     }
   }
 
-  void waitForWorkerThreads() {
-    optQueueNotifyEnd();
-    for (std::thread &Worker : TranslationThreads) {
-      Worker.join();
-    }
-    TranslationThreads.clear();
-
-    // Only notify the emit queue to end after all the translation threads have
-    // ended.
-    emitQueueNotifyEnd();
-    for (std::thread &Worker : EmitterThreads) {
-      Worker.join();
-    }
-    EmitterThreads.clear();
-
-    if (BuildDefs::timers()) {
-      auto Timers = getTimers();
-      for (ThreadContext *TLS : AllThreadContexts)
-        Timers->mergeFrom(TLS->Timers);
-    }
-    if (BuildDefs::dump()) {
-      // Do a separate loop over AllThreadContexts to avoid holding two locks
-      // at once.
-      auto Stats = getStatsCumulative();
-      for (ThreadContext *TLS : AllThreadContexts)
-        Stats->add(TLS->StatsCumulative);
-    }
-  }
+  void waitForWorkerThreads();
 
   /// Translation thread startup routine.
   void translateFunctionsWrapper(ThreadContext *MyTLS) {
@@ -559,11 +547,19 @@ private:
   Ostream *StrError; /// Stream for logging errors.
 
   ICE_CACHELINE_BOUNDARY;
+  // WaitForWorkersLock is a global lock used to update
+  // WaitForWorkerThreadsCalled.
+  GlobalLockType WaitForWorkersLock;
+  // True if WaitForWorkerThreads have been called.

[Jim Stichnoth, 2016/03/25 04:31:04]

True if waitForWorkerThreads() has been called.

[Karl, 2016/03/29 17:35:02]

Done.

+  bool WaitForWorkerThreadsCalled = false;
+
+  ICE_CACHELINE_BOUNDARY;
 
   Intrinsics IntrinsicsInfo;
   // TODO(jpp): move to EmitterContext.
   std::unique_ptr<ELFObjectWriter> ObjectWriter;
-  BoundedProducerConsumerQueue<Cfg> OptQ;
+  static constexpr size_t MaxOptQSize = 1 << 16;
+  BoundedProducerConsumerQueue<OptWorkItem, MaxOptQSize> OptQ;
   BoundedProducerConsumerQueue<EmitterWorkItem> EmitQ;
   // DataLowering is only ever used by a single thread at a time (either in
   // emitItems(), or in IceCompiler::run before the compilation is over.)