Bitcode streaming

include/llvm/Bitcode/BitstreamReader.h

Index: include/llvm/Bitcode/BitstreamReader.h
diff --git a/include/llvm/Bitcode/BitstreamReader.h b/include/llvm/Bitcode/BitstreamReader.h
index 0437f53134dc6cbc237c512467f2a57f0b6c4048..a1b6ed5f97048bb78588dae0474d2afed0093e54 100644
--- a/include/llvm/Bitcode/BitstreamReader.h
+++ b/include/llvm/Bitcode/BitstreamReader.h
@@ -15,14 +15,211 @@
 #ifndef BITSTREAM_READER_H
 #define BITSTREAM_READER_H
 
+#include "llvm/ADT/OwningPtr.h"
 #include "llvm/Bitcode/BitCodes.h"
+#include "llvm/Support/BitcodeStream.h"
 #include <climits>
 #include <string>
 #include <vector>
 
 namespace llvm {
 
-  class Deserializer;
+class Deserializer;
+
+class BitstreamBytes {
+public:
+  BitstreamBytes() { }
+
+  virtual ~BitstreamBytes() {}
+
+  // Returns true if pos is the ending stream position (one byte past the last
+  // valid byte).
+  // May block until Pos bytes have been read, or EOF is reached.
+  virtual bool isEndPos(size_t Pos) = 0;
+
+  // Returns the ending stream position (one byte past the last valid byte).
+  // May block until EOF is reached.
+  virtual size_t getEndPos() = 0;
+
+  // Returns true if seeking to Pos is within the stream or one past the end.
+  // May block until Pos bytes have been read, or EOF is reached.
+  virtual bool canSkipToPos(size_t Pos) = 0;
+
+  // Returns the in memory address of Pos from the beginning of the stream.
+  // May block until Pos bytes have been read, or EOF is reached.
+  // Note that the first byte past the end may be skipped to, but may not have
+  // its address taken.
+  virtual const unsigned char *addressOf(size_t Pos) = 0;
+
+  // Returns the character at Pos from the beginning of the stream.
+  // May block until Pos bytes have been read, or EOF is reached.
+  virtual unsigned char getByte(size_t Pos) = 0;
+
+  // Returns the word at Pos from the beginning of the stream.
+  // (Pos is still counted in bytes, as in the rest of the methods)
+  // May block until Pos bytes have been read, or EOF is reached.
+  virtual uint32_t getWord(size_t Pos) = 0;
+
+private:
+  BitstreamBytes(const BitstreamBytes&);  // DO NOT IMPLEMENT
+  void operator=(const BitstreamBytes&);  // DO NOT IMPLEMENT
+};
+
+class MemoryBitstreamBytes : public BitstreamBytes {
+public:
+  MemoryBitstreamBytes() { }
+
+  MemoryBitstreamBytes(const unsigned char *Start, const unsigned char *End)
+      : FirstChar(Start), LastChar(End) {
+  }
+
+  virtual ~MemoryBitstreamBytes() {}
+
+  virtual bool isEndPos(size_t Pos) {
+    return Pos == static_cast<size_t>(LastChar-FirstChar);
+  }
+
+  virtual size_t getEndPos() {
+    return static_cast<size_t>(LastChar-FirstChar);
+  }
+
+  virtual bool canSkipToPos(size_t Pos) {
+    return Pos <= static_cast<size_t>(LastChar-FirstChar);
+  }
+
+  virtual const unsigned char *addressOf(size_t Pos) {
+    assert(canSkipToPos(Pos) && Pos != static_cast<size_t>(LastChar-FirstChar)
+           && "taking address outside of buffer");
+    return FirstChar + Pos;
+  }
+
+  virtual unsigned char getByte(size_t Pos) {
+    assert(canSkipToPos(Pos) && Pos != static_cast<size_t>(LastChar-FirstChar)
+               && "indexing outside of buffer");
+    return *(FirstChar + Pos);
+  }
+
+  virtual uint32_t getWord(size_t Pos) {
+    assert(canSkipToPos(Pos + 3) &&
+           (Pos + 3) != static_cast<size_t>(LastChar-FirstChar) &&
+           "indexing outside of buffer");
+    const unsigned char *p = FirstChar + Pos;
+    return *p << 0 | 
+      *(p + 1) << 8 |
+      *(p + 2) << 16 |
+      *(p + 3) << 24;
+  }
+
+private:
+  const unsigned char *FirstChar;
+  const unsigned char *LastChar;
+
+  MemoryBitstreamBytes(const MemoryBitstreamBytes&);  // DO NOT IMPLEMENT
+  void operator=(const MemoryBitstreamBytes&);  // DO NOT IMPLEMENT
+};
+
+class LazyBitstreamBytes : public BitstreamBytes {
+public:
+  LazyBitstreamBytes(BitcodeStreamer *streamer) : 
+    Bytes(kChunkSize), Streamer(streamer), BytesRead(0), BytesSkipped(0),
+    BitcodeSize(0), EOFReached(false) {
+    BytesRead = streamer->GetBytes(&Bytes[0], kChunkSize);
+  }
+
+  virtual ~LazyBitstreamBytes() {}
+
+  virtual bool isEndPos(size_t Pos) {
+    if (BitcodeSize) return Pos == BitcodeSize;
+    fetchToPos(Pos);
+    return Pos == BytesRead;
+  }
+
+  virtual size_t getEndPos() {
+    if (BitcodeSize) return BitcodeSize;
+    size_t pos = BytesRead + kChunkSize;
+    // keep fetching until we run out of bytes
+    while (fetchToPos(pos)) pos += kChunkSize;
+    return BitcodeSize;
+  }
+
+  // If the bitcode has a header, then its size is known, and we don't have to
+  // block until we actually want to read it.
+  virtual bool canSkipToPos(size_t Pos) {
+    if (BitcodeSize && Pos <= BitcodeSize) return true;
+    return fetchToPos(Pos) || Pos == BitcodeSize;
+  }
+
+  virtual const unsigned char *addressOf(size_t Pos) {
+    assert(0 && "addressOf inside streaming bitstreams not allowed");
+    return NULL;
+  }
+
+  virtual unsigned char getByte(size_t Pos) {
+    fetchToPos(Pos);
+    assert(Pos < BytesRead && "indexing outside of buffer");
+    return Bytes[Pos + BytesSkipped];
+  }
+
+  virtual uint32_t getWord(size_t Pos) {
+    fetchToPos(Pos + 3);
+    assert(Pos + 3 < BytesRead && "indexing outside of buffer");
+    size_t RealPos = Pos + BytesSkipped;
+    return (Bytes[RealPos + 0] << 0) |
+           (Bytes[RealPos + 1] << 8) |
+           (Bytes[RealPos + 2] << 16) |
+           (Bytes[RealPos + 3] << 24);
+  }
+
+  // Drop s bytes from the front of the stream, pushing the positions of the
+  // remaining bytes down by s. This is used to skip past the bitcode header,
+  // since we don't know a priori if it's present, and we can't put bytes
+  // back into the stream once we've read them.
+  bool dropLeadingBytes(size_t s) {
+    if (BytesRead < s) return true;
+    BytesSkipped = s;
+    BytesRead -= s;
+    return false;
+  }
+
+  void setKnownBitcodeSize(size_t size) {
+    BitcodeSize = size;
+  }
+
+private:
+  const static uint32_t kChunkSize = 4096 * 4;
+  std::vector<unsigned char> Bytes;
+  OwningPtr<BitcodeStreamer> Streamer;
+  size_t BytesRead;   // Bytes read from stream
+  size_t BytesSkipped;// Bytes skipped at start of stream (e.g. wrapper/header)
+  size_t BitcodeSize; // 0 if unknown, set if wrapper was seen or EOF reached
+  bool EOFReached;
+
+  // fetch enough bytes such that Pos can be read or EOF is reached
+  // (i.e. BytesRead > Pos). Return true if Pos can be read.
+  // Unlike most of the functions in BitcodeReader, returns true on success.
+  bool fetchToPos(size_t Pos) {
+    if (EOFReached) return Pos < BitcodeSize;
+    while (Pos >= BytesRead) {
+      Bytes.resize(BytesRead + kChunkSize);
+      size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped],
+                                        kChunkSize);
+      BytesRead += bytes;
+      if (bytes < kChunkSize) {
+        if (BitcodeSize && BytesRead < Pos)
+          assert(0 && "Unexpected short read fetching bitcode");
+        if (BytesRead <= Pos) { // reached EOF/ran out of bytes
+          BitcodeSize = BytesRead;
+          EOFReached = true;
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  LazyBitstreamBytes(const LazyBitstreamBytes&);  // DO NOT IMPLEMENT
+  void operator=(const LazyBitstreamBytes&);  // DO NOT IMPLEMENT
+};
 
 class BitstreamReader {
 public:
@@ -36,9 +233,7 @@ public:
     std::vector<std::pair<unsigned, std::string> > RecordNames;
   };
 private:
-  /// FirstChar/LastChar - This remembers the first and last bytes of the
-  /// stream.
-  const unsigned char *FirstChar, *LastChar;
+  OwningPtr<BitstreamBytes> Bytes;
   
   std::vector<BlockInfo> BlockInfoRecords;
 
@@ -47,10 +242,10 @@ private:
   /// uses this.
   bool IgnoreBlockInfoNames;
   
-  BitstreamReader(const BitstreamReader&);  // NOT IMPLEMENTED
-  void operator=(const BitstreamReader&);  // NOT IMPLEMENTED
+  BitstreamReader(const BitstreamReader&);  // DO NOT IMPLEMENT
+  void operator=(const BitstreamReader&);  // DO NOT IMPLEMENT
 public:
-  BitstreamReader() : FirstChar(0), LastChar(0), IgnoreBlockInfoNames(true) {
+  BitstreamReader() : IgnoreBlockInfoNames(true) {
   }
 
   BitstreamReader(const unsigned char *Start, const unsigned char *End) {
@@ -58,12 +253,17 @@ public:
     init(Start, End);
   }
 
+  BitstreamReader(BitstreamBytes *bsv) {
+    Bytes.reset(bsv);
+  }
+
   void init(const unsigned char *Start, const unsigned char *End) {
-    FirstChar = Start;
-    LastChar = End;
     assert(((End-Start) & 3) == 0 &&"Bitcode stream not a multiple of 4 bytes");
+    Bytes.reset(new MemoryBitstreamBytes(Start, End));
   }
 
+  BitstreamBytes &getBytes() { return *Bytes; }
+
   ~BitstreamReader() {
     // Free the BlockInfoRecords.
     while (!BlockInfoRecords.empty()) {
@@ -76,9 +276,6 @@ public:
     }
   }
   
-  const unsigned char *getFirstChar() const { return FirstChar; }
-  const unsigned char *getLastChar() const { return LastChar; }
-
   /// CollectBlockInfoNames - This is called by clients that want block/record
   /// name information.
   void CollectBlockInfoNames() { IgnoreBlockInfoNames = false; }
@@ -122,7 +319,7 @@ public:
 class BitstreamCursor {
   friend class Deserializer;
   BitstreamReader *BitStream;
-  const unsigned char *NextChar;
+  size_t NextChar;
   
   /// CurWord - This is the current data we have pulled from the stream but have
   /// not returned to the client.
@@ -156,8 +353,7 @@ public:
   }
   
   explicit BitstreamCursor(BitstreamReader &R) : BitStream(&R) {
-    NextChar = R.getFirstChar();
-    assert(NextChar && "Bitstream not initialized yet");
+    NextChar = 0;
     CurWord = 0;
     BitsInCurWord = 0;
     CurCodeSize = 2;
@@ -167,8 +363,7 @@ public:
     freeState();
     
     BitStream = &R;
-    NextChar = R.getFirstChar();
-    assert(NextChar && "Bitstream not initialized yet");
+    NextChar = 0;
     CurWord = 0;
     BitsInCurWord = 0;
     CurCodeSize = 2;
@@ -226,12 +421,12 @@ public:
   unsigned GetAbbrevIDWidth() const { return CurCodeSize; }
   
   bool AtEndOfStream() const {
-    return NextChar == BitStream->getLastChar() && BitsInCurWord == 0;
+    return BitStream->getBytes().isEndPos(NextChar) && BitsInCurWord == 0;
   }
   
   /// GetCurrentBitNo - Return the bit # of the bit we are reading.
   uint64_t GetCurrentBitNo() const {
-    return (NextChar-BitStream->getFirstChar())*CHAR_BIT - BitsInCurWord;
+    return NextChar*CHAR_BIT - BitsInCurWord;
   }
   
   BitstreamReader *getBitStreamReader() {
@@ -246,12 +441,10 @@ public:
   void JumpToBit(uint64_t BitNo) {
     uintptr_t ByteNo = uintptr_t(BitNo/8) & ~3;
     uintptr_t WordBitNo = uintptr_t(BitNo) & 31;
-    assert(ByteNo <= (uintptr_t)(BitStream->getLastChar()-
-                                 BitStream->getFirstChar()) &&
-           "Invalid location");
+    assert(BitStream->getBytes().canSkipToPos(ByteNo) && "Invalid location");
     
     // Move the cursor to the right word.
-    NextChar = BitStream->getFirstChar()+ByteNo;
+    NextChar = ByteNo;
     BitsInCurWord = 0;
     CurWord = 0;
     
@@ -272,7 +465,7 @@ public:
     }
 
     // If we run out of data, stop at the end of the stream.
-    if (NextChar == BitStream->getLastChar()) {
+    if (BitStream->getBytes().isEndPos(NextChar)) {
       CurWord = 0;
       BitsInCurWord = 0;
       return 0;
@@ -281,8 +474,7 @@ public:
     unsigned R = CurWord;
 
     // Read the next word from the stream.
-    CurWord = (NextChar[0] <<  0) | (NextChar[1] << 8) |
-              (NextChar[2] << 16) | (NextChar[3] << 24);
+    CurWord = BitStream->getBytes().getWord(NextChar);
     NextChar += 4;
 
     // Extract NumBits-BitsInCurWord from what we just read.
@@ -376,9 +568,8 @@ public:
 
     // Check that the block wasn't partially defined, and that the offset isn't
     // bogus.
-    const unsigned char *const SkipTo = NextChar + NumWords*4;
-    if (AtEndOfStream() || SkipTo > BitStream->getLastChar() ||
-                           SkipTo < BitStream->getFirstChar())
+    size_t SkipTo = NextChar + NumWords*4;
+    if (AtEndOfStream() || !BitStream->getBytes().canSkipToPos(SkipTo))
       return true;
 
     NextChar = SkipTo;
@@ -409,8 +600,7 @@ public:
     if (NumWordsP) *NumWordsP = NumWords;
 
     // Validate that this block is sane.
-    if (CurCodeSize == 0 || AtEndOfStream() ||
-        NextChar+NumWords*4 > BitStream->getLastChar())
+    if (CurCodeSize == 0 || AtEndOfStream())
       return true;
 
     return false;
@@ -512,24 +702,24 @@ public:
         SkipToWord();  // 32-bit alignment
 
         // Figure out where the end of this blob will be including tail padding.
-        const unsigned char *NewEnd = NextChar+((NumElts+3)&~3);
+        size_t NewEnd = NextChar+((NumElts+3)&~3);
         
         // If this would read off the end of the bitcode file, just set the
         // record to empty and return.
-        if (NewEnd > BitStream->getLastChar()) {
+        if (!BitStream->getBytes().canSkipToPos(NewEnd)) {
           Vals.append(NumElts, 0);
-          NextChar = BitStream->getLastChar();
+          NextChar = BitStream->getBytes().getEndPos();
           break;
         }
         
         // Otherwise, read the number of bytes.  If we can return a reference to
         // the data, do so to avoid copying it.
         if (BlobStart) {
-          *BlobStart = (const char*)NextChar;
+          *BlobStart = (const char*)BitStream->getBytes().addressOf(NextChar);
           *BlobLen = NumElts;
         } else {
           for (; NumElts; ++NextChar, --NumElts)
-            Vals.push_back(*NextChar);
+            Vals.push_back(BitStream->getBytes().getByte(NextChar));
         }
         // Skip over tail padding.
         NextChar = NewEnd;