Bitcode parsing modifications for streaming

lib/Bitcode/Reader/BitcodeReader.cpp

 //===- BitcodeReader.cpp - Internal BitcodeReader implementation ----------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This header defines the BitcodeReader class.
@@ skipping 1510 matching lines @@
 /// RememberAndSkipFunctionBody - When we see the block for a function body,
 /// remember where it is and then skip it.  This lets us lazily deserialize the
 /// functions.
 bool BitcodeReader::RememberAndSkipFunctionBody() {
   // Get the function we are talking about.
   if (FunctionsWithBodies.empty())
     return Error("Insufficient function protos");
 
   Function *Fn = FunctionsWithBodies.back();
   FunctionsWithBodies.pop_back();
+  fprintf(stderr, "skipping %p\n", Fn);
 
   // Save the current stream state.
   uint64_t CurBit = Stream.GetCurrentBitNo();
   DeferredFunctionInfo[Fn] = CurBit;
 
   // Skip over the function block for now.
   if (Stream.SkipBlock())
     return Error("Malformed block record");
   return false;
 }
 
-bool BitcodeReader::ParseModule() {
-  if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID))
+bool BitcodeReader::SuspendModuleParse() {
+  // save our current position
+  NextUnreadBit = Stream.GetCurrentBitNo();
+  // Patch the initializers for globals and aliases up.
+  // TODO(dschuff) this probably only needs to be done once as well
+  ResolveGlobalAndAliasInits();
+  if (!GlobalInits.empty() || !AliasInits.empty())
+    return Error("Malformed global initializer set");
+
+  // Look for intrinsic functions which need to be upgraded at some point
+  // TODO(dschuff): do these intrinsics have bodies? if not, this can be
+  // done only once, before we start dealing with function bodies. if so,
+  // this can probably still be done incrementally instead of this copy/paste
+  // hack
+  for (Module::iterator FI = TheModule->begin(), FE = TheModule->end();
+       FI != FE; ++FI) {
+    Function* NewFn;
+    if (UpgradeIntrinsicFunction(FI, NewFn))
+      UpgradedIntrinsics.push_back(std::make_pair(FI, NewFn));
+  }
+
+  // TODO(dschuff) this probably only needs to be done once (currently it's a

[jasonwkim, 2011/11/03 17:54:09]

I think upstream guys dont like TODO(username) - 
you might want to change it if you are planning to upstream (you should :-)

[(google.com) Derek Schuff, 2011/11/03 18:01:27]

These will go away before I upstream it.

On 2011/11/03 17:54:09, jasonwkim wrote:

+  // noop anyway). find the right place to put it
+  // Look for global variables which need to be renamed.
+  for (Module::global_iterator
+         GI = TheModule->global_begin(), GE = TheModule->global_end();
+       GI != GE; ++GI)
+    UpgradeGlobalVariable(GI);
+  // Force deallocation of memory for these vectors to favor the client that
+  // want lazy deserialization.
+  std::vector<std::pair<GlobalVariable*, unsigned> >().swap(GlobalInits);
+  std::vector<std::pair<GlobalAlias*, unsigned> >().swap(AliasInits);
+  return false;
+}
+
+bool BitcodeReader::ParseModule(bool Resume) {
+  if (Resume)
+    Stream.JumpToBit(NextUnreadBit);
+  else if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID))
     return Error("Malformed block record");
-fprintf(stderr, "module at byte %d\n", Stream.GetCurrentBitNo()*8);
+fprintf(stderr, "module at byte %lu\n", Stream.GetCurrentBitNo()/8);
   SmallVector<uint64_t, 64> Record;
   std::vector<std::string> SectionTable;
   std::vector<std::string> GCTable;
 
   // Read all the records for this module.
   while (!Stream.AtEndOfStream()) {
     unsigned Code = Stream.ReadCode();
     if (Code == bitc::END_BLOCK) {
       if (Stream.ReadBlockEnd())
         return Error("Error at end of module block");
@@ skipping 68 matching lines @@
       case bitc::FUNCTION_BLOCK_ID:
         // If this is the first function body we've seen, reverse the
         // FunctionsWithBodies list.
         if (!HasReversedFunctionsWithBodies) {
           std::reverse(FunctionsWithBodies.begin(), FunctionsWithBodies.end());
           HasReversedFunctionsWithBodies = true;
         }
 
         if (RememberAndSkipFunctionBody())
           return true;
+        if (LazyStreamer) return SuspendModuleParse();
         break;
       }
       continue;
     }
 
     if (Code == bitc::DEFINE_ABBREV) {
       Stream.ReadAbbrevRecord();
       continue;
     }
 
@@ skipping 124 matching lines @@
       FunctionType *FTy =
         dyn_cast<FunctionType>(cast<PointerType>(Ty)->getElementType());
       if (!FTy)
         return Error("Function not a pointer to function type!");
 
       Function *Func = Function::Create(FTy, GlobalValue::ExternalLinkage,
                                         "", TheModule);
 
       Func->setCallingConv(static_cast<CallingConv::ID>(Record[1]));
       bool isProto = Record[2];
+      fprintf(stderr, "func record %p isProto %d\n", Func, isProto);
       Func->setLinkage(GetDecodedLinkage(Record[3]));
       Func->setAttributes(getAttributes(Record[4]));
 
       Func->setAlignment((1 << Record[5]) >> 1);
       if (Record[6]) {
         if (Record[6]-1 >= SectionTable.size())
           return Error("Invalid section ID");
         Func->setSection(SectionTable[Record[6]-1]);
       }
       Func->setVisibility(GetDecodedVisibility(Record[7]));
       if (Record.size() > 8 && Record[8]) {
         if (Record[8]-1 > GCTable.size())
           return Error("Invalid GC ID");
         Func->setGC(GCTable[Record[8]-1].c_str());
       }
       bool UnnamedAddr = false;
       if (Record.size() > 9)
         UnnamedAddr = Record[9];
       Func->setUnnamedAddr(UnnamedAddr);
       ValueList.push_back(Func);
 
       // If this is a function with a body, remember the prototype we are
       // creating now, so that we can match up the body with them later.
-      if (!isProto)
+      if (!isProto) {
         FunctionsWithBodies.push_back(Func);
+        if (LazyStreamer) DeferredFunctionInfo[Func] = 0;
+      }
       break;
     }
     // ALIAS: [alias type, aliasee val#, linkage]
     // ALIAS: [alias type, aliasee val#, linkage, visibility]
     case bitc::MODULE_CODE_ALIAS: {
       if (Record.size() < 3)
         return Error("Invalid MODULE_ALIAS record");
       Type *Ty = getTypeByID(Record[0]);
       if (!Ty) return Error("Invalid MODULE_ALIAS record");
       if (!Ty->isPointerTy())
@@ skipping 60 matching lines @@
     switch (BlockID) {
     case bitc::BLOCKINFO_BLOCK_ID:
       if (Stream.ReadBlockInfoBlock())
         return Error("Malformed BlockInfoBlock");
       break;
     case bitc::MODULE_BLOCK_ID:
       // Reject multiple MODULE_BLOCK's in a single bitstream.
       if (TheModule)
         return Error("Multiple MODULE_BLOCKs in same stream");
       TheModule = M;
-      if (ParseModule())
+      if (ParseModule(false))
         return true;
+      if (LazyStreamer) return false;
       break;
     default:
       if (Stream.SkipBlock())
         return Error("Malformed block record");
       break;
     }
   }
 
   return false;
 }
@@ skipping 941 matching lines @@
 
 
 bool BitcodeReader::isMaterializable(const GlobalValue *GV) const {
   if (const Function *F = dyn_cast<Function>(GV)) {
     return F->isDeclaration() &&
       DeferredFunctionInfo.count(const_cast<Function*>(F));
   }
   return false;
 }
 
+bool BitcodeReader::FindFunctionInStream(Function *F,
+       DenseMap<Function*, uint64_t>::iterator DFII) {
+  //Stream.JumpToBit(NextUnreadBit);
+  while (DFII->second == 0) {
+    if (Stream.AtEndOfStream())
+      return Error("Could not find Function in stream");
+    if(ParseModule(true)) return true;
+  }
+  return false;
+}
+
 bool BitcodeReader::Materialize(GlobalValue *GV, std::string *ErrInfo) {
   Function *F = dyn_cast<Function>(GV);
+  fprintf(stderr, "Materialize %p (%s)\n", F, F->getName().str().c_str());
   // If it's not a function or is already material, ignore the request.
   if (!F || !F->isMaterializable()) return false;
 
   DenseMap<Function*, uint64_t>::iterator DFII = DeferredFunctionInfo.find(F);
   assert(DFII != DeferredFunctionInfo.end() && "Deferred function not found!");
+  if(DFII->second == 0) {
+    if (LazyStreamer && FindFunctionInStream(F, DFII)) return true;
+  }
 
   // Move the bit stream to the saved position of the deferred function body.
   Stream.JumpToBit(DFII->second);
-     fprintf(stderr, "Materialize %s\n", F->getName().str().c_str());
+
   if (ParseFunctionBody(F)) {
     if (ErrInfo) *ErrInfo = ErrorString;
     return true;
   }
 
   // Upgrade any old intrinsic calls in the function.
   for (UpgradedIntrinsicMap::iterator I = UpgradedIntrinsics.begin(),
        E = UpgradedIntrinsics.end(); I != E; ++I) {
     if (I->first != I->second) {
       for (Value::use_iterator UI = I->first->use_begin(),
@@ skipping 185 matching lines @@
   R->setBufferOwned(false);
 
   std::string Triple("");
   if (R->ParseTriple(Triple))
     if (ErrMsg)
       *ErrMsg = R->getErrorString();
 
   delete R;
   return Triple;
 }