Add self-scheduling to threaded translation (vs static)

tools/pnacl-llc/pnacl-llc.cpp

Index: tools/pnacl-llc/pnacl-llc.cpp
diff --git a/tools/pnacl-llc/pnacl-llc.cpp b/tools/pnacl-llc/pnacl-llc.cpp
index 39f0ce314810038c5db18a2ae7f8375b7b0aa02e..d6698f7fef11fd49add4906281085a5054e5a34b 100644
--- a/tools/pnacl-llc/pnacl-llc.cpp
+++ b/tools/pnacl-llc/pnacl-llc.cpp
@@ -45,11 +45,11 @@
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/NaCl.h"
+#include "ThreadedFunctionQueue.h"
 #include "ThreadedStreamingCache.h"
 #include <pthread.h>
 #include <memory>
 
-
 using namespace llvm;
 
 // NOTE: When __native_client__ is defined it means pnacl-llc is built as a
@@ -131,6 +131,11 @@ cl::opt<unsigned>
 SplitModuleCount("split-module",
                  cl::desc("Split PNaCl module"), cl::init(1U));
 
+cl::opt<bool>
+NoSplitModuleDynamic("no-split-module-dynamic",

[Jim Stichnoth, 2014/03/18 20:49:49]

Instead of naming a variable with a negative meaning, could we use something
like SplitModuleDynamic or SplitModuleStatic?

[jvoung (off chromium), 2014/03/18 22:01:17]

Changed to small enum -- is that what you had in mind?

[Jim Stichnoth, 2014/03/18 22:23:13]

I was just thinking of naming it without using the word "No", but this is cool.
:)

+                     cl::desc("Disable self-assigning work to threads."),
+                     cl::init(false));
+
 /// Compile the module provided to pnacl-llc. The file name for reading the
 /// module and other options are taken from globals populated by command-line
 /// option parsing.
@@ -335,6 +340,7 @@ static Module* getModule(StringRef ProgramName, LLVMContext &Context,
 
 static int runCompilePasses(Module *mod,
                             unsigned ModuleIndex,
+                            ThreadedFunctionQueue *FuncQueue,
                             const Triple &TheTriple,
                             TargetMachine &Target,
                             StringRef ProgramName,
@@ -432,11 +438,42 @@ static int runCompilePasses(Module *mod,
     FunctionPassManager* P = static_cast<FunctionPassManager*>(PM.get());
     P->doInitialization();
     unsigned FuncIndex = 0;
-    for (Module::iterator I = mod->begin(), E = mod->end(); I != E; ++I) {
-      if (FuncIndex++ % SplitModuleCount == ModuleIndex) {
-        P->run(*I);
-        CheckABIVerifyErrors(ABIErrorReporter, "Function " + I->getName());
-        I->Dematerialize();
+    if (NoSplitModuleDynamic) {
+      for (Module::iterator I = mod->begin(), E = mod->end(); I != E; ++I) {
+        if (FuncQueue->GrabFunctionStatic(FuncIndex, ModuleIndex)) {
+          P->run(*I);
+          CheckABIVerifyErrors(ABIErrorReporter, "Function " + I->getName());
+          I->Dematerialize();
+        }
+        ++FuncIndex;
+      }
+    } else {
+      unsigned ChunkSize = 0;
+      for (Module::iterator I = mod->begin(), E = mod->end(); I != E; ) {
+        ChunkSize = FuncQueue->RecommendedChunkSize();
+        unsigned NextIndex;
+        bool grabbed = FuncQueue->GrabFunctionDynamic(FuncIndex, ChunkSize,
+                                                      NextIndex);
+        if (grabbed) {
+          while(FuncIndex < NextIndex) {

[Jim Stichnoth, 2014/03/18 20:49:49]

while( ==> while (
http://llvm.org/docs/CodingStandards.html#spaces-before-parentheses

[jvoung (off chromium), 2014/03/18 22:01:17]

Done.

+            P->run(*I);
+            CheckABIVerifyErrors(ABIErrorReporter, "Function " + I->getName());
+            I->Dematerialize();
+            ++FuncIndex;
+            ++I;
+            if (I == E) {

[Jim Stichnoth, 2014/03/18 20:49:49]

Maybe add "&& I != E" to the "while" condition instead of "if(...)break;"?

[jvoung (off chromium), 2014/03/18 22:01:17]

Done. Well at least I didn't use a goto =)

+              break;
+            }
+          }
+        } else {
+          while(FuncIndex < NextIndex) {

[Jim Stichnoth, 2014/03/18 20:49:49]

while( ==> while (
http://llvm.org/docs/CodingStandards.html#spaces-before-parentheses

[jvoung (off chromium), 2014/03/18 22:01:17]

Done.

+            ++FuncIndex;
+            ++I;
+            if (I == E) {

[Jim Stichnoth, 2014/03/18 20:49:49]

Maybe add "&& I != E" to the "while" condition instead of "if(...)break;"?

[jvoung (off chromium), 2014/03/18 22:01:17]

Done.

+              break;
+            }
+          }
+        }
       }
     }
     P->doFinalization();
@@ -455,7 +492,8 @@ static int compileSplitModule(const TargetOptions &Options,
                               const StringRef &ProgramName,
                               Module *GlobalModule,
                               StreamingMemoryObject *StreamingObject,
-                              unsigned ModuleIndex) {
+                              unsigned ModuleIndex,
+                              ThreadedFunctionQueue *FuncQueue) {
   std::auto_ptr<TargetMachine>
     target(TheTarget->createTargetMachine(TheTriple.getTriple(),
                                           MCPU, FeaturesStr, Options,
@@ -495,8 +533,8 @@ static int compileSplitModule(const TargetOptions &Options,
     if (ModuleIndex > 0)
       OutFileName << ".module" << ModuleIndex;
     OwningPtr<tool_output_file> Out
-    (GetOutputStream(TheTarget->getName(), TheTriple.getOS(),
-     OutFileName.str()));
+        (GetOutputStream(TheTarget->getName(), TheTriple.getOS(),
+                         OutFileName.str()));
     if (!Out) return 1;
     formatted_raw_ostream FOS(Out->os());
 #else
@@ -504,7 +542,8 @@ static int compileSplitModule(const TargetOptions &Options,
     ROS.SetBufferSize(1 << 20);
     formatted_raw_ostream FOS(ROS);
 #endif
-    int ret = runCompilePasses(mod, ModuleIndex, TheTriple, Target, ProgramName,
+    int ret = runCompilePasses(mod, ModuleIndex, FuncQueue,
+                               TheTriple, Target, ProgramName,
                                FOS);
     if (ret)
       return ret;
@@ -529,6 +568,7 @@ struct ThreadData {
   Module *GlobalModule;
   StreamingMemoryObject *StreamingObject;
   unsigned ModuleIndex;
+  ThreadedFunctionQueue *FuncQueue;
 };
 
 
@@ -542,7 +582,8 @@ static void *runCompileThread(void *arg) {
                                Data->ProgramName,
                                Data->GlobalModule,
                                Data->StreamingObject,
-                               Data->ModuleIndex);
+                               Data->ModuleIndex,
+                               Data->FuncQueue);
   return reinterpret_cast<void *>(static_cast<intptr_t>(ret));
 }
 
@@ -652,10 +693,13 @@ static int compileModule(StringRef ProgramName) {
 
   SmallVector<pthread_t, 4> Pthreads(SplitModuleCount);
   SmallVector<ThreadData, 4> ThreadDatas(SplitModuleCount);
+  ThreadedFunctionQueue FuncQueue(mod.get(), SplitModuleCount);
 
   if (SplitModuleCount == 1) {
+    NoSplitModuleDynamic = true;  // Do static assignment when single-threaded.
     return compileSplitModule(Options, TheTriple, TheTarget, FeaturesStr,
-                              OLvl, ProgramName, mod.get(), NULL, 0);
+                              OLvl, ProgramName, mod.get(), NULL, 0,
+                              &FuncQueue);
   }
 
   for(unsigned ModuleIndex = 0; ModuleIndex < SplitModuleCount; ++ModuleIndex) {
@@ -668,6 +712,7 @@ static int compileModule(StringRef ProgramName) {
     ThreadDatas[ModuleIndex].GlobalModule = mod.get();
     ThreadDatas[ModuleIndex].StreamingObject = StreamingObject.get();
     ThreadDatas[ModuleIndex].ModuleIndex = ModuleIndex;
+    ThreadDatas[ModuleIndex].FuncQueue = &FuncQueue;
     if (pthread_create(&Pthreads[ModuleIndex], NULL, runCompileThread,
                         &ThreadDatas[ModuleIndex])) {
       report_fatal_error("Failed to create thread");