Bitcode streaming

lib/Bitcode/Reader/BitcodeStream.cpp

+//===---- llvm/Bitcode/Reader/BitcodeStream.cpp -  ----*- C++ -*-===//

[nlewycky, 2011/11/05 00:45:06]

Missing description, needs to be exactly 80 columns.

[(google.com) Derek Schuff, 2011/11/07 17:59:35]

Done.

+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Support for streaming (lazy reading) of bitcode files on disk

[nlewycky, 2011/11/05 00:45:06]

Full sentence.

[(google.com) Derek Schuff, 2011/11/07 17:59:35]

Done.

+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "bitcode-stream"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Bitcode/BitcodeStream.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/system_error.h"
+#include <string>
+#include <cerrno>
+#include <cstdio>
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+#include <fcntl.h>
+using namespace llvm;
+
+// Goals:
+// * BitstreamVector doesn't care about complexities like using
+//   threads/async callbacks to overlap download+compile
+// * Don't want to duplicate bitcode
+// * Don't need to know total bitcode len in advance
+
+// BSV already has random access so this interface is only for in-order
+// streaming (no arbitrary seeking, else we'd have to buffer all the bitcode
+// here in addition to BSV). this also means that if we want the option
+// to free bitcode, BSV/BitcodeReader will implement it.
+
+STATISTIC(NumStreamFetches, "Number of calls to bitcode stream fetch");
+
+namespace {
+
+const static error_code success;
+// Very simple stream backed by a file. Mostly useful for stdin;
+// actual file access is probably still best done with mmap
+class BitcodeFileStreamer : public BitcodeStreamer {
+ int Fd;
+public:
+ BitcodeFileStreamer() : Fd(0) {}
+ virtual ~BitcodeFileStreamer() {
+   close(Fd);
+ }
+ virtual size_t GetBytes(unsigned char *buf, size_t len) {
+   NumStreamFetches++;
+   return read(Fd, buf, len);
+ }
+
+ error_code OpenFile(const std::string &Filename) {
+   int OpenFlags = O_RDONLY;
+ #ifdef O_BINARY

[nlewycky, 2011/11/05 00:45:06]

Sorry, code which does this sort of platform-specific thing needs to live in
lib/Support. (I note that this seems to be a copy of MemoryBuffer::getFile?
Could you refactor?)

[(google.com) Derek Schuff, 2011/11/07 17:59:35]

Actually in that case, probably this whole file should go into Support since
pretty much anything it does will probably be platform-specific (other than
trivial MemoryBuffer wrappers). Actually I'm not sure that the MemoryBuffer
example should stay in here, since it will be dead code. (since we see to want
to keep the existing MemoryBuffer-based ways of creating Modules)

As for refactoring file opening and reading, I'm not sure it's really enough
code to be putting into a general abstraction (if it is, surely there must
already be some "open this filename and return an fd" kind of abstraction
already in LLVM?) in any case, if I were to create a new itnerface, I assume it
would just go in lib/Support, or would it be somewhere more specific?

On 2011/11/05 00:45:06, nlewycky wrote:

+   OpenFlags |= O_BINARY;  // Open input file in binary mode on win32.
+ #endif
+   if (Filename == "-")
+     Fd = 0;
+   else
+     Fd = ::open(Filename.c_str(), OpenFlags);
+  if (Fd == -1) return error_code(errno, posix_category());
+  return success;
+ }
+};
+
+// Streams backed by memory. not used right now because it means the bitcode
+// is duplicated (one copy on the backing store here, and one in BSV)
+class MemoryStreamer : public BitcodeStreamer {
+  const char* Start;
+  const char* End;
+  const char* Cur;
+public:
+  MemoryStreamer(const char* start, size_t len) : Start(start),
+    End(start + len), Cur(start) {}
+  virtual size_t GetBytes(unsigned char *buf, size_t len) {
+    size_t CopyLen = std::min(len, static_cast<size_t>(End - Cur));
+    memcpy(buf, Cur, CopyLen);
+    Cur += CopyLen;
+    return CopyLen;
+  }
+};
+
+class MemoryBufferStreamer : public MemoryStreamer {
+  MemoryBuffer *Buffer;
+  bool IsBufferOwned;
+ public:
+  MemoryBufferStreamer(MemoryBuffer *Buf) :
+    MemoryStreamer(Buffer->getBufferStart(), Buffer->getBufferSize()),
+    Buffer(Buf), IsBufferOwned(false) { }
+  void setBufferOwned(bool owned) { IsBufferOwned = owned; }
+  ~MemoryBufferStreamer() {
+    if (IsBufferOwned) delete Buffer;
+  }
+};
+
+}
+
+namespace llvm {
+BitcodeStreamer* getBitcodeFileStreamer(const std::string &Filename,
+                                        std::string *StrError) {
+  BitcodeFileStreamer* s = new BitcodeFileStreamer();
+  error_code e = s->OpenFile(Filename);
+  if (e != success) {
+    *StrError = std::string() + "Could not open " + Filename + ": "
+        + e.message() + "\n";
+    return NULL;
+  }
+  return s;
+}
+
+BitcodeStreamer* getBitcodeMemoryStreamer(const char *Start, size_t Len) {
+  if (!Start) return NULL;
+  return new MemoryStreamer(Start, Len);
+}
+}